PathEnumeratorBase
using Relativity.Transfer.Resources;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace Relativity.Transfer
{
public abstract class PathEnumeratorBase : IPathEnumerator
{
private readonly IPathValidationProvider pathValidationProvider;
protected ITransferLog Log { get; }
protected IFileSystemService FileSystemService { get; }
internal PathEnumeratorBase(ITransferLog log, IPathValidationProvider pathValidationProvider, IFileSystemService fileSystemService)
{
if (log == null)
throw new ArgumentNullException("log");
if (pathValidationProvider == null)
throw new ArgumentNullException("pathValidationProvider");
if (fileSystemService == null)
throw new ArgumentNullException("fileSystemService");
Log = log;
FileSystemService = fileSystemService;
this.pathValidationProvider = pathValidationProvider;
}
public virtual Task<EnumeratedPathsResult> EnumerateAsync(PathEnumeratorContext context)
{
return EnumerateAsync(context, CancellationToken.None);
}
public virtual async Task<EnumeratedPathsResult> EnumerateAsync(PathEnumeratorContext context, CancellationToken token)
{
if (context == null)
throw new ArgumentNullException("context");
if (context.SearchPaths.Count == 0)
throw new ArgumentException(CoreStrings.TransferPathSearchCountArgumentExceptionMessage, "context");
await OnInitializeAsync(context, token).ConfigureAwait(false);
PathSearchStorage storage = new PathSearchStorage(context, new JsonFileSerializer(), Log, token) {
StartTime = DateTime.Now
};
List<string> list = context.SearchPaths.Select(GetLocalPath).ToList();
pathValidationProvider.Validate(list, context.TargetPath);
List<string>.Enumerator enumerator = list.GetEnumerator();
try {
while (enumerator.MoveNext()) {
string searchPath = enumerator.Current;
Stopwatch sw = Stopwatch.StartNew();
Log.LogInformation("Preparing to search the '{SearchPath}' search path...", searchPath);
if (await OnCheckFolderExistsAsync(searchPath, token).ConfigureAwait(false))
await EnumerateTransferPaths((await OnGetFolderItemAsync(searchPath, token).ConfigureAwait(false)).FullName, context, storage, token).ConfigureAwait(false);
else {
if (!(await OnCheckFileExistsAsync(searchPath, token).ConfigureAwait(false)))
throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, CoreStrings.TransferPathSearchArgumentDoesNotExistExceptionMessage, searchPath));
FileItem fileItem = await OnGetFileItemAsync(searchPath, token).ConfigureAwait(false);
storage.Add(CreateTransferPath(fileItem.FullName, context, null, fileItem, false));
}
sw.Stop();
Log.LogInformation("Successfully searched the '{SearchPath}' search path. Elapsed: {Elapsed}", searchPath, sw.Elapsed);
}
} finally {
((IDisposable)enumerator).Dispose();
}
enumerator = default(List<string>.Enumerator);
EnumeratedPathsResult enumeratedPathsResult = new EnumeratedPathsResult(context.SearchPaths, storage.Paths, storage.PathErrors, storage.TotalDirectories, true, DateTime.Now - storage.StartTime);
Log.LogInformation("Search local path summary - Total files: {TotalFileCount:n0}, Total bytes: {TotalByteCount:n0}, Total error paths: {TotalErrorPaths}", enumeratedPathsResult.TotalFileCount, enumeratedPathsResult.TotalByteCount, enumeratedPathsResult.ErrorPaths.Count());
if (context.ValidateTotalFileCount) {
TransferJobService transferJobService = new TransferJobService(new TransferRequest(), new ClientConfiguration(), Log, token);
foreach (TransferPath item in from x in enumeratedPathsResult.Paths
where x.PathAttributes.HasFlag(TransferPathAttributes.File)
select x) {
transferJobService.Save(item);
}
ITransferStatistics transferStatistics = transferJobService.CalculateStatistics();
if (transferStatistics.TotalRequestFiles != enumeratedPathsResult.TotalFileCount)
throw new TransferException(string.Format(CultureInfo.CurrentCulture, CoreStrings.ValidateTotalFileCountExceptionMessage, enumeratedPathsResult.TotalFileCount, transferStatistics.TotalRequestFiles));
}
return enumeratedPathsResult;
}
public virtual Task<SerializedPathsResult> SerializeAsync(string batchDirectory, PathEnumeratorContext context)
{
return SerializeAsync(batchDirectory, context, CancellationToken.None);
}
[SuppressMessage("Microsoft.Maintainability", "CA1506:AvoidExcessiveClassCoupling", Justification = "This has been reviewed and is OK.")]
public async Task<SerializedPathsResult> SerializeAsync(string batchDirectory, PathEnumeratorContext context, CancellationToken token)
{
if (context == null)
throw new ArgumentNullException("context");
if (context.SearchPaths.Count == 0)
throw new ArgumentException(CoreStrings.TransferPathSearchCountArgumentExceptionMessage, "context");
if (string.IsNullOrEmpty(batchDirectory))
throw new ArgumentNullException("batchDirectory");
if (GlobalSettings.Instance.MaxBytesPerBatch < 1)
throw new InvalidOperationException(CoreStrings.MaxBytesPerBatchExceptionMessage);
if (GlobalSettings.Instance.MaxFilesPerBatch < 1)
throw new InvalidOperationException(CoreStrings.MaxFilesPerBatchExceptionMessage);
await OnInitializeAsync(context, token).ConfigureAwait(false);
BatchSerializer serializer = new BatchSerializer(batchDirectory, context, pathValidationProvider, Log);
List<string> list = context.SearchPaths.Select(GetLocalPath).ToList();
pathValidationProvider.Validate(list, context.TargetPath);
List<string>.Enumerator enumerator = list.GetEnumerator();
try {
while (enumerator.MoveNext()) {
string searchPath = enumerator.Current;
Stopwatch sw = Stopwatch.StartNew();
Log.LogInformation("Preparing to batch the '{SearchPath}' search path...", searchPath);
if (await OnCheckFolderExistsAsync(searchPath, token).ConfigureAwait(false))
await EnumerateTransferPaths((await OnGetFolderItemAsync(searchPath, token).ConfigureAwait(false)).FullName, context, serializer.Storage, token).ConfigureAwait(false);
else {
if (!(await OnCheckFileExistsAsync(searchPath, token).ConfigureAwait(false)))
throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, CoreStrings.TransferPathSearchArgumentDoesNotExistExceptionMessage, searchPath));
FileItem fileItem = await OnGetFileItemAsync(searchPath, token).ConfigureAwait(false);
serializer.AddTransferPath(CreateTransferPath(fileItem.FullName, context, null, fileItem, false));
}
sw.Stop();
Log.LogInformation("Successfully batched the '{SearchPath}' search path. Elapsed: {Elapsed}", searchPath, sw.Elapsed);
}
} finally {
((IDisposable)enumerator).Dispose();
}
enumerator = default(List<string>.Enumerator);
SerializedPathsResult serializedPathsResult = await serializer.CreateBatchAsync().ConfigureAwait(false);
Log.LogInformation("Local serialization paths summary - Total files: {TotalFileCount:n0}, Total bytes: {TotalByteCount:n0}, Total error paths: {TotalErrorPaths}, Total batches: {TotalBatchCount}", serializedPathsResult.TotalFileCount, serializedPathsResult.TotalByteCount, serializedPathsResult.ErrorPaths.Count(), serializedPathsResult.Batches.Count);
return serializedPathsResult;
}
protected virtual int GetMaxDegreeOfFileParallelism(PathEnumeratorContext context)
{
if (context == null)
throw new ArgumentNullException("context");
return Math.Min(context.MaxDegreeOfFileParallelism, 8);
}
protected virtual int GetMaxDegreeOfDirectoryParallelism(PathEnumeratorContext context)
{
if (context == null)
throw new ArgumentNullException("context");
return Math.Min(context.MaxDegreeOfDirectoryParallelism, 8);
}
protected abstract Task OnInitializeAsync(PathEnumeratorContext context, CancellationToken token);
protected abstract Task<bool> OnCheckFileExistsAsync(string path, CancellationToken token);
protected abstract Task<bool> OnCheckFolderExistsAsync(string path, CancellationToken token);
protected abstract Task<bool> OnCheckIsEmptyAsync(string path, CancellationToken token);
[SuppressMessage("Microsoft.Design", "CA1006:DoNotNestGenericTypesInMemberSignatures", Justification = "Nested in a Task type.")]
protected abstract Task<IEnumerable<FileItem>> OnEnumerateFilesAsync(string path, PathEnumeratorContext context, CancellationToken token);
[SuppressMessage("Microsoft.Design", "CA1006:DoNotNestGenericTypesInMemberSignatures", Justification = "Nested in a Task type.")]
protected abstract Task<IEnumerable<FolderItem>> OnEnumerateFoldersAsync(string path, PathEnumeratorContext context, CancellationToken token);
protected abstract Task<FileItem> OnGetFileItemAsync(string path, CancellationToken token);
protected abstract Task<FolderItem> OnGetFolderItemAsync(string path, CancellationToken token);
protected virtual string GetLocalPath(string path)
{
return path;
}
private static StringBuilder BuildTargetPath(string targetPath, string searchPathFolder, string subDirectory, bool isTargetPathUnix)
{
StringBuilder stringBuilder = new StringBuilder(targetPath);
if (stringBuilder.Length > 0) {
if (!string.IsNullOrEmpty(searchPathFolder)) {
stringBuilder.Append("\\");
stringBuilder.Append(searchPathFolder);
}
if (!string.IsNullOrEmpty(subDirectory)) {
stringBuilder.Append("\\");
stringBuilder.Append(subDirectory);
}
if (isTargetPathUnix)
stringBuilder.Replace("\\", "/");
else
stringBuilder.Replace("/", "\\");
}
return stringBuilder;
}
private static TransferPath CreateTransferPath(string searchPath, PathEnumeratorContext context, string searchPathFolder, FileItem fileItem, bool isParentedDirectorySearch)
{
if (string.IsNullOrEmpty(fileItem.FullName))
throw new ArgumentException(string.Format(CultureInfo.CurrentCulture, CoreStrings.FileInfoInvalidExceptionMessage, fileItem.Name), "fileItem");
if (!context.PreserveFolders)
return new TransferPath(fileItem.FullName, TransferPathAttributes.File, context.TargetPath, TransferDirection.Upload, fileItem.Name, fileItem.Length);
string subDirectory = null;
if (isParentedDirectorySearch) {
subDirectory = ((!string.IsNullOrEmpty(fileItem.FolderName)) ? fileItem.FolderName.Replace(searchPath, string.Empty) : string.Empty);
subDirectory = PathHelper.TrimLeadingSlash(subDirectory);
}
string targetPath = BuildTargetPath(context.TargetPath, searchPathFolder, subDirectory, context.IsTargetPathUnix).ToString();
return new TransferPath(fileItem.FullName, TransferPathAttributes.File, targetPath, TransferDirection.Upload, fileItem.Name, fileItem.Length);
}
private void AddTransferPathForEmptyDirectory(string searchPath, PathEnumeratorContext context, PathSearchStorage storage, string searchPathFolder, FolderItem folderItem, bool isParentedDirectorySearch)
{
string targetPath = GetTargetPath(searchPath, context, searchPathFolder, folderItem, isParentedDirectorySearch);
TransferPath transferPath = new TransferPath(folderItem.FullName, TransferPathAttributes.Directory | TransferPathAttributes.Empty, targetPath, TransferDirection.Upload);
PathValidationResult validationResult = pathValidationProvider.Validate(transferPath);
storage.Add(transferPath, validationResult);
}
private static string GetTargetPath(string searchPath, PathEnumeratorContext context, string searchPathFolder, FolderItem folderItem, bool isParentedDirectorySearch)
{
if (context.PreserveFolders) {
string subDirectory = null;
if (isParentedDirectorySearch) {
subDirectory = ((!string.IsNullOrEmpty(folderItem.Parent)) ? folderItem.Parent.Replace(searchPath, string.Empty) : string.Empty);
subDirectory = PathHelper.TrimLeadingSlash(subDirectory);
}
return BuildTargetPath(context.TargetPath, searchPathFolder, subDirectory, context.IsTargetPathUnix).ToString();
}
return context.TargetPath;
}
private void RegisterPathError(PathEnumeratorContext context, PathSearchStorage storage, string path, Exception exception)
{
Log.LogWarning(exception, "A path enumeration failure occurred for path '{Path}'.", path);
storage.PathErrors.Add(new ErrorPath(path, exception.Message));
context.PublishProgress(false, storage);
}
[SuppressMessage("Microsoft.Maintainability", "CA1502:AvoidExcessiveComplexity", Justification = "This has been reviewed.")]
private async Task EnumerateTransferPaths(string searchPath, PathEnumeratorContext context, PathSearchStorage storage, CancellationToken token)
{
<>c__DisplayClass28_0 <>c__DisplayClass28_;
FolderItem searchPathFolderItem2 = <>c__DisplayClass28_.searchPathFolderItem;
FolderItem searchPathFolderItem;
FolderItem folderItem = searchPathFolderItem = await OnGetFolderItemAsync(searchPath, token).ConfigureAwait(false);
string searchPathFolder = string.IsNullOrEmpty(searchPathFolderItem.Parent) ? null : searchPathFolderItem.Name;
ConcurrentStack<FolderItem> folderStack = new ConcurrentStack<FolderItem>(new FolderItem[1] {
searchPathFolderItem
});
ConcurrentStack<FolderItem> subFolderStack = new ConcurrentStack<FolderItem>();
bool flag = context.Configuration.TransferEmptyDirectories;
if (flag)
flag = await OnCheckIsEmptyAsync(searchPath, token).ConfigureAwait(false);
if (flag) {
TransferPath transferPath = new TransferPath(searchPathFolderItem.FullName, TransferPathAttributes.Directory | TransferPathAttributes.Empty, context.TargetPath, TransferDirection.Upload);
PathValidationResult pathValidationResult = pathValidationProvider.Validate(transferPath);
storage.Add(transferPath, pathValidationResult);
if (pathValidationResult.IsOk)
context.PublishProgress(true, storage);
} else {
while (folderStack.Count > 0) {
if (token.IsCancellationRequested)
break;
int maxDegreeOfDirectoryParallelism = GetMaxDegreeOfDirectoryParallelism(context);
int maxDegreeOfFileParallelism = GetMaxDegreeOfFileParallelism(context);
subFolderStack.Clear();
Parallel.ForEach(folderStack, new ParallelOptions {
CancellationToken = token,
MaxDegreeOfParallelism = maxDegreeOfDirectoryParallelism
}, () => 0, delegate(FolderItem nextFolderItem, ParallelLoopState loopState, int localCount) {
if (token.IsCancellationRequested)
loopState.Stop();
string fullName = nextFolderItem.FullName;
List<FolderItem> list;
try {
list = OnEnumerateFoldersAsync(fullName, context, token).ConfigureAwait(false).GetAwaiter().GetResult()
.ToList();
} catch (Exception exception) {
if (ExceptionHelper.IsFatalException(exception) || !ExceptionHelper.IsPathException(exception))
throw;
RegisterPathError(context, storage, fullName, exception);
return 0;
}
List<FileItem> source;
try {
source = OnEnumerateFilesAsync(fullName, context, token).ConfigureAwait(false).GetAwaiter().GetResult()
.ToList();
} catch (Exception exception2) {
if (ExceptionHelper.IsFatalException(exception2) || !ExceptionHelper.IsPathException(exception2))
throw;
RegisterPathError(context, storage, fullName, exception2);
return 0;
}
context.PublishProgress(false, storage);
bool isParentedDirectorySearch = nextFolderItem.Parent != null;
Parallel.ForEach(source, new ParallelOptions {
CancellationToken = token,
MaxDegreeOfParallelism = maxDegreeOfFileParallelism
}, () => 0, delegate(FileItem fileInfo, ParallelLoopState innerLoopState, int innerLocalCount) {
if (token.IsCancellationRequested)
innerLoopState.Stop();
TransferPath transferPath2 = CreateTransferPath(searchPath, context, searchPathFolder, fileInfo, isParentedDirectorySearch);
PathValidationResult validationResult = pathValidationProvider.Validate(transferPath2);
storage.Add(transferPath2, validationResult);
return 0;
}, delegate {
});
bool flag2 = !list.Any() && !source.Any();
if (context.Configuration.TransferEmptyDirectories & flag2)
AddTransferPathForEmptyDirectory(searchPath, context, storage, searchPathFolder, nextFolderItem, isParentedDirectorySearch);
if (nextFolderItem != searchPathFolderItem && !flag2)
storage.IncrementDirectoryCount();
if (context.Option != PathEnumeratorOption.AllDirectories)
return 0;
foreach (FolderItem item in list) {
subFolderStack.Push(item);
}
context.PublishProgress(false, storage);
return 0;
}, delegate {
});
folderStack.Clear();
if (subFolderStack.Count > 0)
folderStack.PushRange(subFolderStack.ToArray());
}
context.PublishProgress(true, storage);
}
}
}
}