Incremental file upload v1

.gitignore

@@ -183,4 +183,7 @@ UpgradeLog*.htm
 *.bim_*.settings
 
 # Microsoft Fakes
-FakesAssemblies/
+FakesAssemblies/
+
+#Sherpa custom tasks
+Sherpa.Installer/customtasks/

Sherpa.Installer/InstallationManager.cs

@@ -235,6 +235,18 @@ var fileName in
                         }
                         break;
                     }
+                    case InstallationOperation.FileWatchUploader:
+                        {
+                            if (useConfigurationForInstall)
+                            {
+                                siteSetupManagerFromConfig.StartFileWatching();
+                            }
+                            else
+                            {
+                                UploadAllChangedFiles(context);
+                            }
+                            break;
+                        }
                     case InstallationOperation.ExportTaxonomy:
                     {
                         ExportTaxonomyGroup();
@@ -260,6 +272,20 @@ var fileName in
             Log.Debug("Completed installation operation");
         }
 
+        private void UploadAllChangedFiles(ClientContext context)
+        {
+            Log.Debug("Starting UploadAllChangedFiles");
+            foreach (var file in Directory.GetFiles(ConfigurationDirectoryPath, "*sitehierarchy.json", SearchOption.AllDirectories))
+            {
+                var sitePersister = new FilePersistanceProvider<ShSiteCollection>(file);
+
+                var siteManager = new SiteSetupManager(context, sitePersister.Load(), _rootPath, _incrementalUpload);
+
+                siteManager.StartFileWatching();
+            }
+
+        }
+
         private void InstallAllTaxonomy(ClientContext context)
         {
             foreach (var file in Directory.GetFiles(ConfigurationDirectoryPath, "*taxonomy.json", SearchOption.AllDirectories))
@@ -499,6 +525,10 @@ public InstallationOperation GetInstallationOperationFromInput(string input)
                 {
                     return InstallationOperation.ForceRecrawl;
                 }
+                case 666:
+                {
+                    return InstallationOperation.FileWatchUploader;
+                }
                 case 0:
                 {
                     return InstallationOperation.ExitApplication;

Sherpa.Installer/InstallationOperation.cs

@@ -15,6 +15,7 @@ public enum InstallationOperation
         ForceRecrawl,
         Invalid,
         ExitApplication,
-        ExecuteCustomTasks
+        ExecuteCustomTasks,
+        FileWatchUploader
     }
 }

Sherpa.Installer/Program.cs

@@ -109,12 +109,15 @@ private static void ShowStartScreenAndExecuteCommand()
             Console.WriteLine("Press 7 to execute custom tasks");
             Console.WriteLine("Press 8 to DELETE all sites (except root site)");
             Console.WriteLine("Press 9 to DELETE all custom site columns and content types");
+            Console.WriteLine("Press 666 to start developer mode: Continuous file upload");
             Console.WriteLine("Press 0 to exit application");
             Console.ForegroundColor = ConsoleColor.Yellow;
             Console.Write("Select a number to perform an operation: ");
             Console.BackgroundColor = ConsoleColor.White;
             Console.ForegroundColor = ConsoleColor.Black;
             var input = Console.ReadLine();
+            Console.BackgroundColor = ConsoleColor.Black;
+            Console.ForegroundColor = ConsoleColor.White;
             Console.ResetColor();
             HandleCommandKeyPress(input);
         }

Sherpa.Library/Deploy/FileListenerAndUploader.cs

@@ -0,0 +1,76 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Reflection;
+using System.Threading;
+using log4net;
+using Sherpa.Library.SiteHierarchy;
+
+namespace Sherpa.Library
+{
+    public class FileListenerAndUploader
+    {
+        private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
+        public SiteSetupManager SetupManager { get; set; }
+        public ManualResetEvent ResetEvent { get; set; }
+        public List<string> ChangedFilesInIteration { get; set; }
+
+        public void CreateFileWatcher(string path, SiteSetupManager setupManager)
+        {
+            SetupManager = setupManager;
+
+            FileSystemWatcher watcher = new FileSystemWatcher();
+            ResetEvent = new ManualResetEvent(false);
+
+            watcher.Path = path;
+            /* Watch for changes in LastAccess and LastWrite times, and 
+               the renaming of files or directories. */
+            watcher.NotifyFilter = NotifyFilters.LastAccess | NotifyFilters.LastWrite
+               | NotifyFilters.FileName | NotifyFilters.DirectoryName;
+
+            watcher.IncludeSubdirectories = true;
+
+            // Add event handlers.
+            watcher.Changed += new FileSystemEventHandler(FileChange);
+            watcher.Created += new FileSystemEventHandler(FileChange);
+
+            // Begin watching.
+            watcher.EnableRaisingEvents = true;
+
+            ChangedFilesInIteration = new List<string>();
+            while (true)
+            {
+                Console.ForegroundColor = ConsoleColor.Green;
+                Console.WriteLine("Watching for changes to files in Sherpa content directory...");
+                Console.ResetColor();
+
+                if (ResetEvent.WaitOne())
+                {
+                    SetupManager.UploadChangedFiles();
+                    ChangedFilesInIteration.Clear();
+                    ResetEvent.Reset();
+                }
+                else
+                {
+                    Console.WriteLine("Waiting timed-out");
+                }
+            }
+            //watcher.WaitForChanged(WatcherChangeTypes.Changed | WatcherChangeTypes.Created | WatcherChangeTypes.Renamed, -1);
+        }
+
+        // Define the event handlers.
+        private void FileChange(object source, FileSystemEventArgs e)
+        {
+            if (e.FullPath.EndsWith(".js") || e.FullPath.EndsWith(".css") || e.FullPath.EndsWith(".txt") ||
+                e.FullPath.EndsWith(".html") || e.FullPath.EndsWith(".webpart") || e.FullPath.EndsWith(".png"))
+            {
+                if (!ChangedFilesInIteration.Contains(e.FullPath))
+                {
+                    Log.InfoFormat("File {0} changed and a new upload is starting", e.Name);
+                    ChangedFilesInIteration.Add(e.FullPath);
+                    ResetEvent.Set();
+                }
+            }
+        }
+    }
+}

Sherpa.Library/Sherpa.Library.csproj

@@ -110,6 +110,7 @@
     <Compile Include="CustomTasks\CustomTasksManager.cs" />
     <Compile Include="CustomTasks\ICustomTasksManager.cs" />
     <Compile Include="Deploy\DeployManager.cs" />
+    <Compile Include="Deploy\FileListenerAndUploader.cs" />
     <Compile Include="Deploy\IDeployManager.cs" />
     <Compile Include="SharePointContext.cs" />
     <Compile Include="SiteHierarchy\ComposedLookManager.cs" />

Sherpa.Library/SiteHierarchy/ContentUploadManager.cs

@@ -21,27 +21,21 @@ public class ContentUploadManager
     {
         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
         private static Dictionary<string, DateTime> LastUpload = new Dictionary<string, DateTime>();
-        private static bool IncrementalUpload;
 
         private readonly string _contentDirectoryPath;
         public ContentUploadManager(string rootConfigurationPath)
         {
             _contentDirectoryPath = rootConfigurationPath;
         }
-        public ContentUploadManager(string rootConfigurationPath, bool incrementalUpload)
-        {
-            _contentDirectoryPath = rootConfigurationPath;
-            IncrementalUpload = incrementalUpload;
-        }
-        public void UploadFilesInFolder(ClientContext context, Web web, List<ShContentFolder> contentFolders)
+        public void UploadFilesInFolder(ClientContext context, Web web, List<ShContentFolder> contentFolders, bool incrementalUpload)
         {
             foreach (ShContentFolder folder in contentFolders)
             {
-                UploadFilesInFolder(context, web, folder);
+                UploadFilesInFolder(context, web, folder, incrementalUpload);
             }
         }
 
-        public void UploadFilesInFolder(ClientContext context, Web web, ShContentFolder contentFolder)
+        public void UploadFilesInFolder(ClientContext context, Web web, ShContentFolder contentFolder, bool incrementalUpload)
         {
             Log.Info("Uploading files from contentfolder " + contentFolder.FolderName);
 
@@ -115,41 +109,15 @@ public void UploadFilesInFolder(ClientContext context, Web web, ShContentFolder
             var files = Directory.GetFiles(configRootFolder, "*", SearchOption.AllDirectories)
                 .Where(file => !excludedFileExtensions.Contains(Path.GetExtension(file).ToLower())).ToList();
 
-            if (IncrementalUpload)
+            if (incrementalUpload)
             {
                 files = files.Where(f =>!LastUpload.ContainsKey(contentFolder.FolderName) || new FileInfo(f).LastWriteTimeUtc > LastUpload[contentFolder.FolderName]).ToList();
             }
 
             int filesUploaded = 0;
             foreach (string filePath in files)
             {
-                var pathToFileFromRootFolder = filePath.Replace(configRootFolder.TrimEnd(new []{'\\'}) + "\\", "");
-                var fileName = Path.GetFileName(pathToFileFromRootFolder);
-
-                if (!string.IsNullOrEmpty(contentFolder.PropertiesFile) && contentFolder.PropertiesFile == fileName)
-                {
-                    Log.DebugFormat("Skipping file upload of {0} since it's used as a configuration file", fileName);
-                    continue;
-                }
-                Log.DebugFormat("Uploading file {0} to {1}", fileName, contentFolder.ListUrl);
-                var fileUrl = GetFileUrl(uploadTargetFolder, pathToFileFromRootFolder, filePropertiesCollection);
-                web.CheckOutFile(fileUrl);
-
-                var newFile = new FileCreationInformation
-                {
-                    Content = System.IO.File.ReadAllBytes(filePath),
-                    Url = fileUrl,
-                    Overwrite = true
-                };
-                File uploadFile = rootFolder.Files.Add(newFile);
-
-                context.Load(uploadFile);
-                context.Load(uploadFile.ListItemAllFields.ParentList, l => l.ForceCheckout, l => l.EnableMinorVersions, l => l.EnableModeration);
-                context.ExecuteQuery();
-
-                ApplyFileProperties(context, filePropertiesCollection, uploadFile);
-                uploadFile.PublishFileToLevel(FileLevel.Published);
-                context.ExecuteQuery();
+                UploadAndPublishSingleFile(context, web, configRootFolder, contentFolder, uploadTargetFolder, rootFolder, filePropertiesCollection, filePath);
 
                 filesUploaded++;
             }
@@ -174,6 +142,42 @@ public void UploadFilesInFolder(ClientContext context, Web web, ShContentFolder
 
         }
 
+        private void UploadAndPublishSingleFile(ClientContext context, Web web, string configRootFolder, ShContentFolder contentFolder, string uploadTargetFolder, Folder rootFolder, List<ShFileProperties> filePropertiesCollection, string filePath)
+        {
+            var pathToFileFromRootFolder = filePath.Replace(configRootFolder.TrimEnd(new[] { '\\' }) + "\\", "");
+            var fileName = Path.GetFileName(pathToFileFromRootFolder);
+
+            if (!string.IsNullOrEmpty(contentFolder.PropertiesFile) && contentFolder.PropertiesFile == fileName)
+            {
+                Log.DebugFormat("Skipping file upload of {0} since it's used as a configuration file", fileName);
+                return;
+            }
+            Log.DebugFormat("Uploading file {0} to {1}", fileName, contentFolder.ListUrl);
+            var fileUrl = GetFileUrl(uploadTargetFolder, pathToFileFromRootFolder, filePropertiesCollection);
+            web.CheckOutFile(fileUrl);
+
+            var newFile = new FileCreationInformation
+            {
+                Content = System.IO.File.ReadAllBytes(filePath),
+                Url = fileUrl,
+                Overwrite = true
+            };
+            File uploadFile = rootFolder.Files.Add(newFile);
+
+            context.Load(uploadFile);
+            context.Load(uploadFile.ListItemAllFields.ParentList, l => l.ForceCheckout, l => l.EnableMinorVersions, l => l.EnableModeration);
+            context.ExecuteQuery();
+
+            var reloadedFile = web.GetFileByServerRelativeUrl(fileUrl);
+            context.Load(reloadedFile);
+            context.ExecuteQuery();
+
+            ApplyFileProperties(context, filePropertiesCollection, reloadedFile);
+
+            uploadFile.PublishFileToLevel(FileLevel.Published);
+            context.ExecuteQuery();
+        }
+
         private string GetFileUrl(string uploadTargetFolder, string pathToFileFromRootFolder,
             IEnumerable<ShFileProperties> filePropertiesCollection)
         {

Sherpa.Library/SiteHierarchy/SiteSetupManager.cs

@@ -4,6 +4,7 @@
 using System.Reflection;
 using log4net;
 using Microsoft.SharePoint.Client;
+using System;
 
 namespace Sherpa.Library.SiteHierarchy
 {
@@ -21,11 +22,14 @@ public class SiteSetupManager : ISiteSetupManager
         private CustomActionsManager CustomActionsManager { get; set; }
         private PermissionManager PermissionManager { get; set; }
         private ComposedLookManager ComposedLookManager { get; set; }
-
+        private FileListenerAndUploader FileListenerAndUploader { get; set; }
+        private bool IncrementalUpload { get; set; }
+        private string ContentConfigurationPath { get; set; }
         public SiteSetupManager(ClientContext clientContext, ShSiteCollection configurationSiteCollection, string rootConfigurationPath, bool incrementalUpload)
         {
             ConfigurationSiteCollection = configurationSiteCollection;
             ClientContext = clientContext;
+            IncrementalUpload = incrementalUpload;
 
             FeatureManager = new FeatureManager();
             QuicklaunchManager = new QuicklaunchManager();
@@ -34,9 +38,10 @@ public SiteSetupManager(ClientContext clientContext, ShSiteCollection configurat
             CustomActionsManager = new CustomActionsManager();
             PermissionManager = new PermissionManager();
             ComposedLookManager = new ComposedLookManager();
+            FileListenerAndUploader = new FileListenerAndUploader();
 
-            var contentConfigurationPath = Path.Combine(rootConfigurationPath, "content");
-            ContentUploadManager = new ContentUploadManager(contentConfigurationPath, incrementalUpload);
+            ContentConfigurationPath = Path.Combine(rootConfigurationPath, "content");
+            ContentUploadManager = new ContentUploadManager(ContentConfigurationPath);
         }
         public void SetupSites()
         {
@@ -60,7 +65,7 @@ private void EnsureAndConfigureWebAndActivateFeatures(ClientContext context, Web
             ListManager.CreateLists(context, webToConfigure, configWeb.Lists);
             QuicklaunchManager.CreateQuicklaunchNodes(context, webToConfigure, configWeb.Quicklaunch);
             PropertyManager.SetProperties(context, webToConfigure, configWeb.Properties);
-            ContentUploadManager.UploadFilesInFolder(context, webToConfigure, configWeb.ContentFolders);
+            ContentUploadManager.UploadFilesInFolder(context, webToConfigure, configWeb.ContentFolders, IncrementalUpload);
             ComposedLookManager.SetComposedLook(context, configWeb, webToConfigure, configWeb.ComposedLook);
             SearchNavigationManager.CreateSearchNavigationNodes(context, webToConfigure, configWeb.SearchNavigation);
             SetWelcomePageUrlIfConfigured(context, webToConfigure, configWeb);
@@ -177,5 +182,27 @@ private static void DeleteWeb(ClientContext clientContext, Web web)
             web.DeleteObject();
             clientContext.ExecuteQuery();
         }
+
+        public void StartFileWatching()
+        {
+            UploadFilesInWeb(ClientContext, null, ConfigurationSiteCollection.RootWeb);
+            FileListenerAndUploader.CreateFileWatcher(ContentConfigurationPath, this);
+        }
+        public void UploadChangedFiles()
+        {
+            UploadFilesInWeb(ClientContext, null, ConfigurationSiteCollection.RootWeb);
+        }
+
+        private void UploadFilesInWeb(ClientContext context, Web parentWeb, ShWeb configWeb)
+        {
+            Log.Info("Looking for updated files in web " + configWeb.Url);
+            var webToConfigure = EnsureWeb(context, parentWeb, configWeb);
+            ContentUploadManager.UploadFilesInFolder(context, webToConfigure, configWeb.ContentFolders, IncrementalUpload);
+
+            foreach (ShWeb subWeb in configWeb.Webs)
+            {
+                UploadFilesInWeb(context, webToConfigure, subWeb);
+            }
+        }
     }
 }