BulkImageFileImporter
using kCura.EDDS.WebAPI.BulkImportManagerBase;
using kCura.EDDS.WebAPI.FieldManagerBase;
using kCura.WinEDDS.Api;
using kCura.WinEDDS.Service;
using kCura.WinEDDS.Service.Replacement;
using Microsoft.VisualBasic;
using Microsoft.VisualBasic.CompilerServices;
using Monitoring;
using Relativity.DataExchange;
using Relativity.DataExchange.Data;
using Relativity.DataExchange.Io;
using Relativity.DataExchange.Logger;
using Relativity.DataExchange.Logging;
using Relativity.DataExchange.Media;
using Relativity.DataExchange.Process;
using Relativity.DataExchange.Service;
using Relativity.DataExchange.Transfer;
using Relativity.Logging;
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
namespace kCura.WinEDDS
{
public class BulkImageFileImporter : ImportTapiBase
{
public delegate void EndRunEventHandler (string runID);
public delegate void FatalErrorEventEventHandler (string message, Exception ex);
public delegate void StatusMessageEventHandler (StatusEventArgs args);
public delegate void ReportErrorEventEventHandler (IDictionary row);
protected IImageReader _imageReader;
protected IFieldQuery _fieldQuery;
protected IProductionManager _productionManager;
protected IBulkImportManager _bulkImportManager;
private int _folderID;
private int _productionArtifactID;
private ImportOverwriteType _overwrite;
private string _filePath;
private long _recordCount;
private bool _replaceFullText;
private int? _importBatchSize;
private int? _jobCompleteBatchSize;
private int? _importBatchVolume;
private int? _minimumBatchSize;
private bool _autoNumberImages;
private bool _copyFilesToRepository;
private string _defaultDestinationFolderPath;
private CaseInfo _caseInfo;
private int _overlayArtifactID;
private global::Relativity.DataExchange.Service.ExecutionSource _executionSource;
private long _lastRunMetadataImport;
[CompilerGenerated]
[AccessedThroughProperty("_processContext")]
private ProcessContext __processContext;
protected Field _keyFieldDto;
protected bool _fullTextStorageIsInSql;
private StreamWriter _bulkLoadFileWriter;
private StreamWriter _dataGridFileWriter;
private string _uploadKey;
private string _uploadDataGridKey;
private string _localRunId;
private string _runId;
private ImageLoadFile _settings;
private int _batchCount;
private int _jobCompleteImageCount;
private int _jobCompleteMetadataCount;
private int _errorCount;
private string _errorMessageFileLocation;
private string _errorRowsFileLocation;
private Hashtable _fileIdentifierLookup;
private Guid _processID;
[CompilerGenerated]
private int _MaxNumberOfErrorsInGrid;
private long _totalValidated;
private long _totalProcessed;
private long _startLineNumber;
private Timekeeper2 _timekeeper;
private bool _doRetryLogic;
private ClientSideErrorCollection _verboseErrorCollection;
private List<Tuple<ImageRecord, string>> _prePushErrors;
private bool _cancelledByUser;
[CompilerGenerated]
private IImageValidator __imageValidator;
[CompilerGenerated]
private ITiffValidator __tiffValidator;
[CompilerGenerated]
private IFileInspector __fileInspector;
[CompilerGenerated]
private bool _SkipExtractedTextEncodingCheck;
[CompilerGenerated]
private bool _OIFileIdMapped;
[CompilerGenerated]
private string _OIFileIdColumnName;
[CompilerGenerated]
private string _OIFileTypeColumnName;
[CompilerGenerated]
private string _FileNameColumn;
[CompilerGenerated]
private bool _DisableImageTypeValidation;
[CompilerGenerated]
private bool _DisableImageLocationValidation;
[CompilerGenerated]
private bool _DisableUserSecurityCheck;
[CompilerGenerated]
private ImportAuditLevel _AuditLevel;
[CompilerGenerated]
private List<int> _BatchSizeHistoryList;
[CompilerGenerated]
private EndRunEventHandler EndRunEvent;
[CompilerGenerated]
private FatalErrorEventEventHandler FatalErrorEventEvent;
[CompilerGenerated]
private StatusMessageEventHandler StatusMessageEvent;
[CompilerGenerated]
private ReportErrorEventEventHandler ReportErrorEventEvent;
private virtual ProcessContext _processContext {
[CompilerGenerated]
get {
return __processContext;
}
[MethodImpl(MethodImplOptions.Synchronized)]
[CompilerGenerated]
set {
EventHandler<CancellationRequestEventArgs> value2 = _processObserver_CancelImport;
EventHandler<ExportErrorEventArgs> value3 = _processContext_ExportServerErrorsEvent;
EventHandler<ExportErrorEventArgs> value4 = _processContext_ExportErrorFileEvent;
EventHandler<ExportErrorEventArgs> value5 = _processContext_ExportErrorReportEvent;
EventHandler<ParentFormClosingEventArgs> value6 = _processContext_ParentFormClosingEvent;
ProcessContext _processContext = __processContext;
if (_processContext != null) {
_processContext.CancellationRequest -= value2;
_processContext.ExportServerErrors -= value3;
_processContext.ExportErrorFile -= value4;
_processContext.ExportErrorReport -= value5;
_processContext.ParentFormClosing -= value6;
}
__processContext = value;
_processContext = __processContext;
if (_processContext != null) {
_processContext.CancellationRequest += value2;
_processContext.ExportServerErrors += value3;
_processContext.ExportErrorFile += value4;
_processContext.ExportErrorReport += value5;
_processContext.ParentFormClosing += value6;
}
}
}
public int MaxNumberOfErrorsInGrid { get; set; }
private IImageValidator _imageValidator { get; set; }
private ITiffValidator _tiffValidator { get; set; }
private IFileInspector _fileInspector { get; set; }
public bool SkipExtractedTextEncodingCheck { get; set; }
public bool OIFileIdMapped { get; set; }
public string OIFileIdColumnName { get; set; }
public string OIFileTypeColumnName { get; set; }
public string FileNameColumn { get; set; }
internal virtual long TotalRecords {
get {
if (_recordCount <= 0)
_recordCount = _imageReader.CountRecords().GetValueOrDefault();
return _recordCount;
}
}
internal long CompletedRecords => base.TotalTransferredFilesCount;
public bool DisableImageTypeValidation { get; set; }
public bool DisableImageLocationValidation { get; set; }
public bool DisableUserSecurityCheck { get; set; }
public ImportAuditLevel AuditLevel { get; set; }
public List<int> BatchSizeHistoryList { get; }
internal string FilePath {
set {
_filePath = value;
}
}
internal bool IsCancelledByUser => _cancelledByUser;
public bool HasErrors => _errorCount > 0;
public string RunId => _runId;
protected bool Continue {
get {
if (_imageReader.HasMoreRecords)
return base.ShouldImport;
return false;
}
}
public TapiClient UploadConnection => base.FileTapiClient;
public string ErrorLogFileName => _errorMessageFileLocation;
protected override int CurrentLineNumber => _imageReader.CurrentRecordNumber;
protected virtual int MinimumBatchSize {
get {
if (!_minimumBatchSize.HasValue)
_minimumBatchSize = AppSettings.Instance.MinBatchSize;
return _minimumBatchSize.Value;
}
set {
_minimumBatchSize = value;
}
}
protected virtual int ImportBatchSize {
get {
if (!_importBatchSize.HasValue)
_importBatchSize = AppSettings.Instance.ImportBatchSize;
return _importBatchSize.Value;
}
set {
_importBatchSize = ((value > MinimumBatchSize) ? value : MinimumBatchSize);
}
}
protected int JobCompleteBatchSize {
get {
if (!_jobCompleteBatchSize.HasValue)
_jobCompleteBatchSize = AppSettings.Instance.JobCompleteBatchSize;
return _jobCompleteBatchSize.Value;
}
set {
_jobCompleteBatchSize = ((value > MinimumBatchSize) ? value : MinimumBatchSize);
}
}
protected int ImportBatchVolume {
get {
if (!_importBatchVolume.HasValue)
_importBatchVolume = AppSettings.Instance.ImportBatchMaxVolume;
return _importBatchVolume.Value;
}
set {
_importBatchVolume = value;
}
}
protected virtual int NumberOfRetries => AppSettings.Instance.IoErrorNumberOfRetries;
protected virtual int WaitTimeBetweenRetryAttempts => AppSettings.Instance.IoErrorWaitTimeInSeconds;
protected virtual bool BatchResizeEnabled => AppSettings.Instance.DynamicBatchResizingOn;
public event EndRunEventHandler EndRun {
[CompilerGenerated]
add {
EndRunEventHandler endRunEventHandler = EndRunEvent;
EndRunEventHandler endRunEventHandler2;
do {
endRunEventHandler2 = endRunEventHandler;
EndRunEventHandler value2 = (EndRunEventHandler)Delegate.Combine(endRunEventHandler2, value);
endRunEventHandler = Interlocked.CompareExchange(ref EndRunEvent, value2, endRunEventHandler2);
} while ((object)endRunEventHandler != endRunEventHandler2);
}
[CompilerGenerated]
remove {
EndRunEventHandler endRunEventHandler = EndRunEvent;
EndRunEventHandler endRunEventHandler2;
do {
endRunEventHandler2 = endRunEventHandler;
EndRunEventHandler value2 = (EndRunEventHandler)Delegate.Remove(endRunEventHandler2, value);
endRunEventHandler = Interlocked.CompareExchange(ref EndRunEvent, value2, endRunEventHandler2);
} while ((object)endRunEventHandler != endRunEventHandler2);
}
}
public event FatalErrorEventEventHandler FatalErrorEvent {
[CompilerGenerated]
add {
FatalErrorEventEventHandler fatalErrorEventEventHandler = FatalErrorEventEvent;
FatalErrorEventEventHandler fatalErrorEventEventHandler2;
do {
fatalErrorEventEventHandler2 = fatalErrorEventEventHandler;
FatalErrorEventEventHandler value2 = (FatalErrorEventEventHandler)Delegate.Combine(fatalErrorEventEventHandler2, value);
fatalErrorEventEventHandler = Interlocked.CompareExchange(ref FatalErrorEventEvent, value2, fatalErrorEventEventHandler2);
} while ((object)fatalErrorEventEventHandler != fatalErrorEventEventHandler2);
}
[CompilerGenerated]
remove {
FatalErrorEventEventHandler fatalErrorEventEventHandler = FatalErrorEventEvent;
FatalErrorEventEventHandler fatalErrorEventEventHandler2;
do {
fatalErrorEventEventHandler2 = fatalErrorEventEventHandler;
FatalErrorEventEventHandler value2 = (FatalErrorEventEventHandler)Delegate.Remove(fatalErrorEventEventHandler2, value);
fatalErrorEventEventHandler = Interlocked.CompareExchange(ref FatalErrorEventEvent, value2, fatalErrorEventEventHandler2);
} while ((object)fatalErrorEventEventHandler != fatalErrorEventEventHandler2);
}
}
public event StatusMessageEventHandler StatusMessage {
[CompilerGenerated]
add {
StatusMessageEventHandler statusMessageEventHandler = StatusMessageEvent;
StatusMessageEventHandler statusMessageEventHandler2;
do {
statusMessageEventHandler2 = statusMessageEventHandler;
StatusMessageEventHandler value2 = (StatusMessageEventHandler)Delegate.Combine(statusMessageEventHandler2, value);
statusMessageEventHandler = Interlocked.CompareExchange(ref StatusMessageEvent, value2, statusMessageEventHandler2);
} while ((object)statusMessageEventHandler != statusMessageEventHandler2);
}
[CompilerGenerated]
remove {
StatusMessageEventHandler statusMessageEventHandler = StatusMessageEvent;
StatusMessageEventHandler statusMessageEventHandler2;
do {
statusMessageEventHandler2 = statusMessageEventHandler;
StatusMessageEventHandler value2 = (StatusMessageEventHandler)Delegate.Remove(statusMessageEventHandler2, value);
statusMessageEventHandler = Interlocked.CompareExchange(ref StatusMessageEvent, value2, statusMessageEventHandler2);
} while ((object)statusMessageEventHandler != statusMessageEventHandler2);
}
}
public event ReportErrorEventEventHandler ReportErrorEvent {
[CompilerGenerated]
add {
ReportErrorEventEventHandler reportErrorEventEventHandler = ReportErrorEventEvent;
ReportErrorEventEventHandler reportErrorEventEventHandler2;
do {
reportErrorEventEventHandler2 = reportErrorEventEventHandler;
ReportErrorEventEventHandler value2 = (ReportErrorEventEventHandler)Delegate.Combine(reportErrorEventEventHandler2, value);
reportErrorEventEventHandler = Interlocked.CompareExchange(ref ReportErrorEventEvent, value2, reportErrorEventEventHandler2);
} while ((object)reportErrorEventEventHandler != reportErrorEventEventHandler2);
}
[CompilerGenerated]
remove {
ReportErrorEventEventHandler reportErrorEventEventHandler = ReportErrorEventEvent;
ReportErrorEventEventHandler reportErrorEventEventHandler2;
do {
reportErrorEventEventHandler2 = reportErrorEventEventHandler;
ReportErrorEventEventHandler value2 = (ReportErrorEventEventHandler)Delegate.Remove(reportErrorEventEventHandler2, value);
reportErrorEventEventHandler = Interlocked.CompareExchange(ref ReportErrorEventEvent, value2, reportErrorEventEventHandler2);
} while ((object)reportErrorEventEventHandler != reportErrorEventEventHandler2);
}
}
public BulkImageFileImporter(int folderID, ImageLoadFile args, ProcessContext context, IIoReporter reporter, ILog logger, Guid processID, bool doRetryLogic, CancellationTokenSource tokenSource, Func<string> correlationIdFunc, global::Relativity.DataExchange.Service.ExecutionSource executionSource = global::Relativity.DataExchange.Service.ExecutionSource.Unknown)
: base(reporter, logger, tokenSource, correlationIdFunc)
{
_lastRunMetadataImport = 0;
_fullTextStorageIsInSql = true;
_uploadKey = "";
_uploadDataGridKey = "";
Guid guid = Guid.NewGuid();
_localRunId = guid.ToString().Replace("-", "_");
_runId = "";
_batchCount = 0;
_jobCompleteImageCount = 0;
_jobCompleteMetadataCount = 0;
_errorCount = 0;
_errorMessageFileLocation = "";
_errorRowsFileLocation = "";
MaxNumberOfErrorsInGrid = AppSettings.Instance.DefaultMaxErrorCount;
_timekeeper = new Timekeeper2();
_verboseErrorCollection = new ClientSideErrorCollection();
_prePushErrors = new List<Tuple<ImageRecord, string>>();
_cancelledByUser = false;
_imageValidator = new ImageValidator();
_tiffValidator = new TiffValidator();
_fileInspector = new FileInspector();
DisableImageTypeValidation = AppSettings.Instance.DisableImageTypeValidation;
DisableImageLocationValidation = AppSettings.Instance.DisableImageLocationValidation;
AuditLevel = Config.AuditLevel;
_executionSource = executionSource;
CorrelationIdFunc = correlationIdFunc;
_doRetryLogic = doRetryLogic;
InitializeManagers(args);
if ((object)_bulkImportManager.GetType() == typeof(KeplerBulkImportManager)) {
guid = Guid.NewGuid();
_runId = guid.ToString().Replace("-", "_");
}
string str = "\\EDDS" + Conversions.ToString(args.CaseInfo.ArtifactID) + "\\";
if (Operators.CompareString(args.SelectedCasePath, "", false) == 0)
_defaultDestinationFolderPath = args.CaseDefaultPath.TrimEnd(new char[1] {
'\\'
}) + str;
else
_defaultDestinationFolderPath = args.SelectedCasePath.TrimEnd(new char[1] {
'\\'
}) + str;
InitializeUploaders(args);
_folderID = folderID;
_productionArtifactID = args.ProductionArtifactID;
base.Statistics.ImportObjectType = (TelemetryConstants.ImportObjectType)Conversions.ToInteger(Interaction.IIf(_productionArtifactID == 0, TelemetryConstants.ImportObjectType.Image, TelemetryConstants.ImportObjectType.ProductionImage));
InitializeDTOs(args);
if (args.Overwrite.IsNullOrEmpty())
_overwrite = ImportOverwriteType.Append;
else
_overwrite = (ImportOverwriteType)Conversions.ToInteger(Enum.Parse(typeof(ImportOverwriteType), args.Overwrite, true));
_replaceFullText = args.ReplaceFullText;
_processContext = context;
_copyFilesToRepository = args.CopyFilesToDocumentRepository;
base.ShouldImport = true;
_autoNumberImages = args.AutoNumberImages;
_caseInfo = args.CaseInfo;
_settings = args;
_processID = processID;
_startLineNumber = args.StartLineNumber;
_overlayArtifactID = args.IdentityFieldId;
_BatchSizeHistoryList = new List<int>();
if (args.ReplaceFullText)
_fullTextStorageIsInSql = !_fieldQuery.RetrieveAllAsDocumentFieldCollection(args.CaseInfo.ArtifactID, 10).FullText.EnableDataGrid;
}
protected virtual void InitializeUploaders(ImageLoadFile args)
{
IFileIO fileIO = ManagerFactory.CreateFileIO(args.Credential, args.CookieContainer, GetCorrelationId);
UploadTapiBridgeParameters2 uploadTapiBridgeParameters = new UploadTapiBridgeParameters2();
uploadTapiBridgeParameters.BcpFileTransfer = false;
uploadTapiBridgeParameters.AsperaBcpRootFolder = string.Empty;
uploadTapiBridgeParameters.Application = AppSettings.Instance.ApplicationName;
uploadTapiBridgeParameters.ClientRequestId = Guid.NewGuid();
uploadTapiBridgeParameters.Credentials = args.Credential;
uploadTapiBridgeParameters.AsperaDocRootLevels = AppSettings.Instance.TapiAsperaNativeDocRootLevels;
uploadTapiBridgeParameters.AsperaDatagramSize = AppSettings.Instance.TapiAsperaDatagramSize;
uploadTapiBridgeParameters.FileShare = args.CaseInfo.DocumentPath;
uploadTapiBridgeParameters.ForceAsperaClient = AppSettings.Instance.TapiForceAsperaClient;
uploadTapiBridgeParameters.ForceClientCandidates = AppSettings.Instance.TapiForceClientCandidates;
uploadTapiBridgeParameters.ForceFileShareClient = AppSettings.Instance.TapiForceFileShareClient;
uploadTapiBridgeParameters.ForceHttpClient = (AppSettings.Instance.ForceWebUpload || AppSettings.Instance.TapiForceHttpClient);
uploadTapiBridgeParameters.LargeFileProgressEnabled = AppSettings.Instance.TapiLargeFileProgressEnabled;
uploadTapiBridgeParameters.LogConfigFile = AppSettings.Instance.LogConfigXmlFileName;
uploadTapiBridgeParameters.MaxFilesPerFolder = fileIO.RepositoryVolumeMax();
uploadTapiBridgeParameters.MaxInactivitySeconds = AppSettings.Instance.TapiMaxInactivitySeconds;
uploadTapiBridgeParameters.MaxJobParallelism = AppSettings.Instance.TapiMaxJobParallelism;
uploadTapiBridgeParameters.MaxJobRetryAttempts = NumberOfRetries;
uploadTapiBridgeParameters.MinDataRateMbps = AppSettings.Instance.TapiMinDataRateMbps;
uploadTapiBridgeParameters.SubmitApmMetrics = AppSettings.Instance.TapiSubmitApmMetrics;
uploadTapiBridgeParameters.TargetPath = _defaultDestinationFolderPath;
uploadTapiBridgeParameters.TargetDataRateMbps = AppSettings.Instance.TapiTargetDataRateMbps;
uploadTapiBridgeParameters.TimeoutSeconds = AppSettings.Instance.HttpTimeoutSeconds;
uploadTapiBridgeParameters.TransferLogDirectory = AppSettings.Instance.TapiTransferLogDirectory;
uploadTapiBridgeParameters.WaitTimeBetweenRetryAttempts = WaitTimeBetweenRetryAttempts;
uploadTapiBridgeParameters.WebCookieContainer = args.CookieContainer;
uploadTapiBridgeParameters.WebServiceUrl = AppSettings.Instance.WebApiServiceUrl;
uploadTapiBridgeParameters.WorkspaceId = args.CaseInfo.ArtifactID;
uploadTapiBridgeParameters.PermissionErrorsRetry = AppSettings.Instance.PermissionErrorsRetry;
uploadTapiBridgeParameters.PreserveFileTimestamps = AppSettings.Instance.TapiPreserveFileTimestamps;
uploadTapiBridgeParameters.BadPathErrorsRetry = AppSettings.Instance.TapiBadPathErrorsRetry;
uploadTapiBridgeParameters.FileNotFoundErrorsDisabled = AppSettings.Instance.TapiFileNotFoundErrorsDisabled;
uploadTapiBridgeParameters.FileNotFoundErrorsRetry = AppSettings.Instance.TapiFileNotFoundErrorsRetry;
UploadTapiBridgeParameters2 uploadTapiBridgeParameters2 = uploadTapiBridgeParameters.ShallowCopy();
uploadTapiBridgeParameters2.BcpFileTransfer = true;
uploadTapiBridgeParameters2.AsperaBcpRootFolder = AppSettings.Instance.TapiAsperaBcpRootFolder;
uploadTapiBridgeParameters2.FileShare = fileIO.GetBcpSharePath(args.CaseInfo.ArtifactID);
uploadTapiBridgeParameters2.SortIntoVolumes = false;
uploadTapiBridgeParameters2.ForceHttpClient |= AppSettings.Instance.TapiForceBcpHttpClient;
uploadTapiBridgeParameters2.PreserveFileTimestamps = false;
CreateTapiBridges(uploadTapiBridgeParameters, uploadTapiBridgeParameters2, args.WebApiCredential.TokenProvider, new RelativityManagerServiceFactory());
}
protected virtual void InitializeDTOs(ImageLoadFile args)
{
IFieldManager fieldManager = ManagerFactory.CreateFieldManager(args.Credential, args.CookieContainer, GetCorrelationId);
if (_productionArtifactID > 0)
_keyFieldDto = fieldManager.Read(args.CaseInfo.ArtifactID, args.BeginBatesFieldArtifactID);
else if (args.IdentityFieldId > 0) {
_keyFieldDto = fieldManager.Read(args.CaseInfo.ArtifactID, args.IdentityFieldId);
} else {
int fieldID = _fieldQuery.RetrieveAllAsDocumentFieldCollection(args.CaseInfo.ArtifactID, 10).IdentifierFields()[0].FieldID;
_keyFieldDto = fieldManager.Read(args.CaseInfo.ArtifactID, fieldID);
}
}
protected virtual void InitializeManagers(ImageLoadFile args)
{
_fieldQuery = ManagerFactory.CreateFieldQuery(args.Credential, args.CookieContainer, GetCorrelationId);
_productionManager = ManagerFactory.CreateProductionManager(args.Credential, args.CookieContainer, GetCorrelationId);
_bulkImportManager = ManagerFactory.CreateBulkImportManager(args.Credential, args.CookieContainer, GetCorrelationId);
}
public void ReadFile()
{
ReadFile(_filePath);
}
private void ProcessList(List<ImageRecord> al, ref long status, string bulkLoadFilePath, string dataGridFilePath)
{
if (al.Count != 0) {
ProcessDocument(al, status);
al.Clear();
status = 0;
if (checked(_bulkLoadFileWriter.BaseStream.Length + _dataGridFileWriter.BaseStream.Length > ImportBatchVolume || _batchCount > ImportBatchSize - 1))
TryPushImageBatch(bulkLoadFilePath, dataGridFilePath, false, _jobCompleteImageCount >= JobCompleteBatchSize, _jobCompleteMetadataCount >= JobCompleteBatchSize);
}
}
public MassImportResults RunBulkImport(OverwriteType overwrite, bool useBulk)
{
int numberOfRetries = NumberOfRetries;
int num = numberOfRetries;
MassImportResults result = new MassImportResults();
checked {
while (num > 0) {
try {
result = BulkImport(overwrite, useBulk);
return result;
} catch (Exception ex) {
ProjectData.SetProjectError(ex);
Exception ex2 = ex;
num--;
if (num == 0) {
LogFatal(ex2, "The image bulk import service call failed and exceeded the max retry attempts.");
throw;
}
if (ImportTapiBase.IsTimeoutException(ex2)) {
LogError(ex2, "A fatal SQL or HTTP timeout error has occurred bulk importing the image batch.");
throw;
}
if (!base.ShouldImport)
throw;
if (ImportTapiBase.IsBulkImportSqlException(ex2)) {
LogFatal(ex2, "A fatal SQL error has occurred bulk importing the image batch.");
throw;
}
if (ImportTapiBase.IsInsufficientPermissionsForImportException(ex2)) {
LogFatal(ex2, "A fatal insufficient permissions error has occurred bulk importing the image batch.");
throw;
}
LogWarning(ex2, "A serious error has occurred bulk importing the image batch. Retry info: {Count} of {TotalRetry}.", numberOfRetries - num, numberOfRetries);
RaiseWarningAndPause(ex2, WaitTimeBetweenRetryAttempts, numberOfRetries - num, numberOfRetries);
ProjectData.ClearProjectError();
}
}
return result;
}
}
private MassImportResults BulkImport(OverwriteType overwrite, bool useBulk)
{
ImageLoadInfo settingsObject = GetSettingsObject();
settingsObject.UseBulkDataImport = useBulk;
settingsObject.Overlay = overwrite;
settingsObject.Billable = _settings.Billable;
if (_productionArtifactID != 0)
return _bulkImportManager.BulkImportProductionImage(_caseInfo.ArtifactID, settingsObject, _productionArtifactID, _copyFilesToRepository);
return _bulkImportManager.BulkImportImage(_caseInfo.ArtifactID, settingsObject, _copyFilesToRepository);
}
protected virtual void LowerBatchLimits()
{
int importBatchSize = ImportBatchSize;
checked {
ImportBatchSize -= 100;
base.Statistics.BatchSize = ImportBatchSize;
BatchSizeHistoryList.Add(ImportBatchSize);
LogWarning("Lowered the image batch limits from {OldBatchSize} to {NewBatchSize}.", importBatchSize, ImportBatchSize);
}
}
private ImageLoadInfo GetSettingsObject()
{
return new ImageLoadInfo {
RunID = _runId,
DestinationFolderArtifactID = _folderID,
BulkFileName = _uploadKey,
DataGridFileName = _uploadDataGridKey,
KeyFieldArtifactID = _keyFieldDto.ArtifactID,
Repository = _defaultDestinationFolderPath,
UploadFullText = _replaceFullText,
DisableUserSecurityCheck = DisableUserSecurityCheck,
AuditLevel = AuditLevel,
OverlayArtifactID = _overlayArtifactID,
ExecutionSource = _executionSource
};
}
private void TryPushImageBatch(string bulkLoadFilePath, string dataGridFilePath, bool isFinal, bool shouldCompleteImageJob, bool shouldCompleteMetadataJob)
{
_bulkLoadFileWriter.Close();
_dataGridFileWriter.Close();
_fileIdentifierLookup.Clear();
if ((shouldCompleteImageJob | isFinal) & (_jobCompleteImageCount > 0)) {
_jobCompleteImageCount = 0;
AwaitPendingPhysicalFileUploadsForJob();
}
checked {
try {
if (base.ShouldImport && _copyFilesToRepository && base.FileTapiBridge.TransfersPending) {
AwaitPendingPhysicalFileUploadsForBatch();
base.JobCounter++;
}
DateTime now = DateTime.Now;
long ticks = now.Ticks;
if (base.ShouldImport)
PushImageBatch(bulkLoadFilePath, dataGridFilePath, shouldCompleteMetadataJob, isFinal);
ImportStatistics statistics;
ImportStatistics importStatistics = statistics = base.Statistics;
TimeSpan fileWaitDuration = statistics.FileWaitDuration;
now = DateTime.Now;
importStatistics.FileWaitDuration = fileWaitDuration + new TimeSpan(Math.Max(now.Ticks - ticks, 1));
} catch (Exception ex) {
ProjectData.SetProjectError(ex);
Exception ex2 = ex;
if (!BatchResizeEnabled || !ImportTapiBase.IsTimeoutException(ex2) || !base.ShouldImport) {
if (base.ShouldImport && !BatchResizeEnabled)
LogFatal(ex2, "Pushing the image batch failed but lowering the batch and performing a retry is disabled.");
if (base.ShouldImport && BatchResizeEnabled)
LogFatal(ex2, "Pushing the image batch failed but lowering the batch isn't supported because the error isn't timeout related.");
throw;
}
LogWarning(ex2, "A SQL or HTTP timeout error has occurred bulk importing the image batch and the batch will be resized.");
int importBatchSize = ImportBatchSize;
LowerBatchLimits();
RaiseWarningAndPause(ex2, WaitTimeBetweenRetryAttempts);
if (!base.ShouldImport)
throw;
LowerBatchSizeAndRetry(bulkLoadFilePath, dataGridFilePath, importBatchSize);
ProjectData.ClearProjectError();
}
DeleteFiles(bulkLoadFilePath, dataGridFilePath);
if (!isFinal)
try {
_bulkLoadFileWriter = new StreamWriter(bulkLoadFilePath, false, Encoding.Unicode);
_dataGridFileWriter = new StreamWriter(dataGridFilePath, false, Encoding.Unicode);
} catch (Exception ex3) {
ProjectData.SetProjectError(ex3);
Exception exception = ex3;
LogWarning(exception, "Failed to create new image bulk load files. Preparing to retry...");
_bulkLoadFileWriter = new StreamWriter(bulkLoadFilePath, false, Encoding.Unicode);
_dataGridFileWriter = new StreamWriter(dataGridFilePath, false, Encoding.Unicode);
ProjectData.ClearProjectError();
}
UpdateStatisticsSnapshot(DateTime.Now, true);
}
}
protected void LowerBatchSizeAndRetry(string oldBulkLoadFilePath, string dataGridFilePath, int totalRecords)
{
string tempFileName = TempFileBuilder.GetTempFileName("rel-native");
string text = "þþKþþ\r\n";
Queue<char> queue = new Queue<char>();
int num = 0;
long charactersSuccessfullyProcessed = 0;
bool flag = false;
int num2 = 1;
checked {
while (totalRecords > num && !flag && base.ShouldImport) {
int num3 = 0;
long num4 = 0;
using (TextReader textReader = CreateStreamReader(oldBulkLoadFilePath))
using (TextWriter textWriter = CreateStreamWriter(tempFileName)) {
AdvanceStream(textReader, charactersSuccessfullyProcessed);
int num5 = ImportBatchSize;
while (!flag && num3 < num5) {
char c = Strings.ChrW(textReader.Read());
queue.Enqueue(c);
if (queue.Count > text.Length)
queue.Dequeue();
if (Operators.CompareString(new string(queue.ToArray()), text, false) == 0) {
textWriter.Flush();
num3++;
}
textWriter.Write(c);
num4++;
flag = (textReader.Peek() == -1);
if (num3 >= num5 && textReader.Peek() == 48)
num5++;
}
textWriter.Flush();
}
try {
num = DoLogicAndPushImageBatch(totalRecords, num, tempFileName, dataGridFilePath, ref charactersSuccessfullyProcessed, num3, num4);
} catch (Exception ex) {
ProjectData.SetProjectError(ex);
Exception ex2 = ex;
if (num2 >= NumberOfRetries || !BatchResizeEnabled || !ImportTapiBase.IsTimeoutException(ex2) || !base.ShouldImport) {
global::Relativity.DataExchange.Io.FileSystem.Instance.File.Delete(tempFileName);
throw;
}
LowerBatchLimits();
RaiseWarningAndPause(ex2, WaitTimeBetweenRetryAttempts, num2, NumberOfRetries);
if (!base.ShouldImport)
throw;
num2++;
flag = false;
ProjectData.ClearProjectError();
}
}
global::Relativity.DataExchange.Io.FileSystem.Instance.File.Delete(tempFileName);
}
}
protected virtual int DoLogicAndPushImageBatch(int totalRecords, int recordsProcessed, string bulkLocation, string dataGridLocation, ref long charactersSuccessfullyProcessed, int i, long charactersProcessed)
{
_batchCount = i;
checked {
RaiseStatusEvent(EventType2.Warning, "Begin processing sub-batch of size " + Conversions.ToString(i) + ".", (long)Math.Round(unchecked((double)checked(_totalValidated + _totalProcessed) / 2)));
PushImageBatch(bulkLocation, dataGridLocation, false, true);
RaiseStatusEvent(EventType2.Warning, "End processing sub-batch of size " + Conversions.ToString(i) + ". " + Conversions.ToString(recordsProcessed) + " of " + Conversions.ToString(totalRecords) + " in the original batch processed", (long)Math.Round(unchecked((double)checked(_totalValidated + _totalProcessed) / 2)));
recordsProcessed += i;
charactersSuccessfullyProcessed += charactersProcessed;
return recordsProcessed;
}
}
private void DeleteFiles(string bulkFilePath, string datagridFilePath)
{
global::Relativity.DataExchange.Io.FileSystem.Instance.File.Delete(bulkFilePath);
global::Relativity.DataExchange.Io.FileSystem.Instance.File.Delete(datagridFilePath);
}
protected virtual TextWriter CreateStreamWriter(string tmpLocation)
{
return new StreamWriter(tmpLocation, false, Encoding.Unicode);
}
protected virtual TextReader CreateStreamReader(string outputPath)
{
return new StreamReader(outputPath, Encoding.Unicode);
}
private void AdvanceStream(TextReader sr, long count)
{
checked {
if (count > 0) {
long num = count - 1;
for (long num2 = 0; num2 <= num; num2++) {
sr.Read();
}
}
}
}
public unsafe void PushImageBatch(string bulkLoadFilePath, string dataGridFilePath, bool shouldCompleteJob, bool lastRun)
{
ManagePrePushErrors();
checked {
DateTime now;
if (_lastRunMetadataImport > 0) {
ImportStatistics statistics;
ImportStatistics importStatistics = statistics = base.Statistics;
TimeSpan metadataWaitDuration = statistics.MetadataWaitDuration;
now = DateTime.Now;
importStatistics.MetadataWaitDuration = metadataWaitDuration + new TimeSpan(now.Ticks - _lastRunMetadataImport);
}
if (_batchCount == 0) {
if (_jobCompleteMetadataCount > 0) {
_jobCompleteMetadataCount = 0;
AwaitPendingBulkLoadFileUploadsForJob();
}
} else {
if (shouldCompleteJob & (_jobCompleteMetadataCount > 0)) {
_jobCompleteMetadataCount = 0;
AwaitPendingBulkLoadFileUploadsForJob();
}
_batchCount = 0;
ImportStatistics statistics;
(statistics = base.Statistics).MetadataTransferredBytes = statistics.MetadataTransferredBytes + (GetFileLength(bulkLoadFilePath, true) + GetFileLength(dataGridFilePath, true));
UploadTapiBridge2 bulkLoadTapiBridge = base.BulkLoadTapiBridge;
Guid guid = Guid.NewGuid();
_uploadKey = bulkLoadTapiBridge.AddPath(bulkLoadFilePath, guid.ToString(), 1);
UploadTapiBridge2 bulkLoadTapiBridge2 = base.BulkLoadTapiBridge;
guid = Guid.NewGuid();
_uploadDataGridKey = bulkLoadTapiBridge2.AddPath(dataGridFilePath, guid.ToString(), 2);
base.MetadataFilesCount += 2;
ref int jobCompleteMetadataCount;
*(ref jobCompleteMetadataCount = ref _jobCompleteMetadataCount) = jobCompleteMetadataCount + 2;
if (lastRun)
AwaitPendingBulkLoadFileUploadsForJob();
else
AwaitPendingBulkLoadFileUploadsForBatch();
now = DateTime.Now;
_lastRunMetadataImport = now.Ticks;
OverwriteType overwrite;
switch (_overwrite) {
case ImportOverwriteType.AppendOverlay:
overwrite = OverwriteType.Both;
break;
case ImportOverwriteType.Overlay:
overwrite = OverwriteType.Overlay;
break;
default:
overwrite = OverwriteType.Append;
break;
}
if (base.ShouldImport) {
now = DateTime.Now;
long ticks = now.Ticks;
MassImportResults massImportResults = RunBulkImport(overwrite, true);
base.Statistics.ProcessMassImportResults(massImportResults);
if (string.IsNullOrWhiteSpace(_runId)) {
_runId = massImportResults.RunID;
base.Logger.LogWarning("CorrelationId mapping [{localId}] - [{runId}]", new object[2] {
_localRunId,
_runId
});
}
now = DateTime.Now;
long ticks2 = now.Ticks - ticks;
TimeSpan timeSpan = new TimeSpan(ticks2);
(statistics = base.Statistics).MassImportDuration = statistics.MassImportDuration + timeSpan;
(statistics = base.Statistics).BatchCount = statistics.BatchCount + 1;
base.Logger.LogInformation("Duration of mass import processing: {durationInMilliseconds}, batch: {numberOfBatch}", new object[2] {
timeSpan.TotalMilliseconds,
base.Statistics.BatchCount
});
ManageErrors();
base.TotalTransferredFilesCount = base.FileTapiProgressCount;
BatchInformation batchInformation = new BatchInformation {
OrdinalNumber = base.Statistics.BatchCount,
NumberOfRecords = massImportResults.FilesProcessed,
MassImportDuration = timeSpan
};
base.OnBatchCompleted(batchInformation);
}
}
}
}
public virtual IImageReader GetImageReader()
{
return new OpticonFileReader(_folderID, _settings, null, default(Guid), _doRetryLogic);
}
public void AdvanceRecord()
{
_imageReader.AdvanceRecord();
}
public void ReadFile(string path)
{
_timekeeper.MarkStart("TOTAL");
using (_logger.LogImportContextPushProperties(new LogContext(_localRunId, _settings.CaseInfo.ArtifactID))) {
_logger.LogUserContextInformation("Start import process", _settings.Credential);
string tempFileName = TempFileBuilder.GetTempFileName("rel-native");
string tempFileName2 = TempFileBuilder.GetTempFileName("rel-datagrid");
_fileIdentifierLookup = new Hashtable();
_totalProcessed = 0;
_totalValidated = 0;
base.TotalTransferredFilesCount = 0;
base.JobCounter = 1;
base.FileTapiProgressCount = 0;
DeleteFiles(tempFileName, tempFileName2);
_bulkLoadFileWriter = new StreamWriter(tempFileName, false, Encoding.Unicode);
_dataGridFileWriter = new StreamWriter(tempFileName2, false, Encoding.Unicode);
try {
_timekeeper.MarkStart("ReadFile_Init");
_filePath = path;
_imageReader = GetImageReader();
_imageReader.Initialize();
_recordCount = _imageReader.CountRecords().GetValueOrDefault();
RaiseStatusEvent(EventType2.Progress, "Begin Image Upload", 0);
RaiseStatusEvent(EventType2.ResetStartTime, "", 0);
List<ImageRecord> list = new List<ImageRecord>();
long status = 0;
_timekeeper.MarkEnd("ReadFile_Init");
_timekeeper.MarkStart("ReadFile_Main");
OnTapiClientChanged();
LogInformation("Preparing to import images via WinEDDS.");
base.Statistics.BatchSize = ImportBatchSize;
if (_productionArtifactID != 0)
_productionManager.DoPreImportProcessing(_caseInfo.ArtifactID, _productionArtifactID);
while (Continue) {
if (CurrentLineNumber < _startLineNumber) {
AdvanceRecord();
checked {
base.FileTapiProgressCount++;
}
} else {
RaiseStatusEvent(EventType2.Count, string.Empty, 0);
ImageRecord imageRecord = _imageReader.GetImageRecord();
imageRecord.OriginalIndex = _imageReader.CurrentRecordNumber;
if (imageRecord.IsNewDoc)
ProcessList(list, ref status, tempFileName, tempFileName2);
status |= (long)ProcessImageLine(imageRecord);
try {
ValidateImageRecord(imageRecord);
list.Add(imageRecord);
} catch (ImageDataValidationException ex) {
ProjectData.SetProjectError(ex);
ImageDataValidationException ex2 = ex;
_verboseErrorCollection.AddError(imageRecord.OriginalIndex, ex2);
_prePushErrors.Add(new Tuple<ImageRecord, string>(imageRecord, ex2.Message));
ProjectData.ClearProjectError();
}
if (!Continue) {
ProcessList(list, ref status, tempFileName, tempFileName2);
break;
}
}
}
_timekeeper.MarkEnd("ReadFile_Main");
_timekeeper.MarkStart("ReadFile_Cleanup");
TryPushImageBatch(tempFileName, tempFileName2, true, true, false);
LogInformation("Successfully imported {ImportCount} images via WinEDDS.", base.FileTapiProgressCount);
CompleteSuccess();
_timekeeper.MarkEnd("ReadFile_Cleanup");
_timekeeper.MarkEnd("TOTAL");
_timekeeper.GenerateCsvReportItemsAsRows("_winedds_image", "C:\\");
} catch (Exception ex3) {
ProjectData.SetProjectError(ex3);
Exception exception = ex3;
LogFatal(exception, "A serious unexpected error has occurred importing images.");
CompleteError(exception);
ProjectData.ClearProjectError();
} finally {
EndRunEvent?.Invoke(string.IsNullOrEmpty(_runId) ? _localRunId : _runId);
LogStatistics();
_timekeeper.MarkStart("ReadFile_CleanupTempTables");
DestroyTapiBridges();
CleanupTempTables();
_logger.LogUserContextInformation("Import process completed", _settings.Credential);
_timekeeper.MarkEnd("ReadFile_CleanupTempTables");
}
}
}
private void CompleteSuccess()
{
if (_imageReader != null)
_imageReader.Close();
if (_productionArtifactID != 0)
_productionManager.DoPostImportProcessing(base.FileTapiBridge.WorkspaceId, _productionArtifactID);
if (base.CancellationToken.IsCancellationRequested)
OnWriteStatusMessage(EventType2.Status, "Job has been finalized", 0, 0);
else
OnWriteStatusMessage(EventType2.Status, "End Image Upload");
}
private void CompleteError(Exception exception)
{
try {
_bulkLoadFileWriter.Close();
_dataGridFileWriter.Close();
} catch (Exception ex) {
ProjectData.SetProjectError(ex);
Exception exception2 = ex;
LogWarning(exception2, "Failed to close the image load file writers before raising the image import fatal error.");
ProjectData.ClearProjectError();
}
try {
if (_imageReader != null)
_imageReader.Close();
} catch (Exception ex2) {
ProjectData.SetProjectError(ex2);
Exception exception3 = ex2;
LogWarning(exception3, "Failed to close the image reader before raising the image import fatal error.");
ProjectData.ClearProjectError();
}
try {
if (!string.IsNullOrEmpty(_runId))
ManageErrors();
else
LogInformation("There was no any Bulk Import execution so there are no errors to get from DB.");
} catch (Exception ex3) {
ProjectData.SetProjectError(ex3);
Exception exception4 = ex3;
LogWarning(exception4, "Failed to manage errors before raising the image import fatal error.");
ProjectData.ClearProjectError();
}
RaiseFatalError(exception);
}
private void ProcessDocument(List<ImageRecord> al, long status)
{
GetImagesForDocument(al, status);
ImportStatistics statistics;
(statistics = base.Statistics).DocumentsCount = checked(statistics.DocumentsCount + 1);
}
public unsafe ImportStatus ProcessImageLine(ImageRecord imageRecord)
{
checked {
ref long totalValidated;
*(ref totalValidated = ref _totalValidated) = totalValidated + 1;
if (Operators.CompareString(imageRecord.BatesNumber.Trim(), "", false) != 0) {
string text = GetFileLocation(imageRecord);
if (!DisableImageLocationValidation) {
string existingFilePath = GetExistingFilePath(text, true);
if (string.IsNullOrEmpty(existingFilePath)) {
RaiseStatusEvent(EventType2.Error, $"""{imageRecord.FileLocation}""", (long)Math.Round(unchecked((double)checked(_totalValidated + _totalProcessed) / 2)));
return ImportStatus.FileSpecifiedDne;
}
if (!string.Equals(text, existingFilePath)) {
RaiseStatusEvent(EventType2.Warning, $"""{text}""{existingFilePath}""", (long)Math.Round(unchecked((double)checked(_totalValidated + _totalProcessed) / 2)));
text = existingFilePath;
}
}
ImportStatus result = ImportStatus.Pending;
try {
if (!DisableImageTypeValidation) {
ImageValidationResult imageValidationResult = _imageValidator.IsImageValid(text, _tiffValidator, _fileInspector);
if (!imageValidationResult.IsValid)
throw new ImageFileValidationException(imageValidationResult.Message);
}
RaiseStatusEvent(EventType2.Status, $"""{imageRecord.FileLocation}""", (long)Math.Round(unchecked((double)checked(_totalValidated + _totalProcessed) / 2)));
} catch (ImageFileValidationException ex) {
ProjectData.SetProjectError(ex);
ImageFileValidationException ex2 = ex;
LogError(ex2, "Failed to validate the {Path} image.", text.Secure());
result = ImportStatus.InvalidImageFormat;
_verboseErrorCollection.AddError(imageRecord.OriginalIndex, ex2);
ProjectData.ClearProjectError();
} catch (Exception ex3) {
ProjectData.SetProjectError(ex3);
Exception exception = ex3;
LogFatal(exception, "Unexpected failure to validate the {Path} image file.", text.Secure());
throw;
}
return result;
}
RaiseStatusEvent(EventType2.Error, "No image file or identifier specified on line.", (long)Math.Round(unchecked((double)checked(_totalValidated + _totalProcessed) / 2)));
return ImportStatus.NoImageSpecifiedOnLine;
}
}
private void ValidateImageRecord(ImageRecord record)
{
string batesNumber = record.BatesNumber;
if (batesNumber != null && batesNumber.IndexOf(",") > -1)
throw new ImageDataValidationException("Bates number contains unsupported char ','");
string fileName = record.FileName;
if (fileName != null && fileName.IndexOf(",") > -1)
throw new ImageDataValidationException("File name contains unsupported char ','");
string fileLocation = record.FileLocation;
if (fileLocation != null && fileLocation.IndexOf(",") > -1)
throw new ImageDataValidationException("File location contains unsupported char ','");
}
public static string GetFileLocation(ImageRecord record)
{
string text = record.FileLocation;
if (Operators.CompareString(text, "", false) != 0 && Operators.CompareString(Conversions.ToString(text[0]), "\\", false) == 0 && Operators.CompareString(Conversions.ToString(text[1]), "\\", false) != 0)
text = "." + text;
return text;
}
private void GetImagesForDocument(List<ImageRecord> lines, long status)
{
AutoNumberImages(lines);
bool flag = false;
foreach (ImageRecord line in lines) {
if (_fileIdentifierLookup.ContainsKey(line.BatesNumber.Trim()))
flag = true;
else
_fileIdentifierLookup.Add(line.BatesNumber.Trim(), line.BatesNumber.Trim());
}
checked {
if (flag)
status += 8192;
ImageRecord imageRecord = lines[0];
ArrayList arrayList = new ArrayList();
string batesNumber = imageRecord.BatesNumber;
long offset = 0;
int num = lines.Count - 1;
for (int i = 0; i <= num; i++) {
if (!base.ShouldImport)
break;
imageRecord = lines[i];
string text = GetFileLocation(imageRecord);
string existingFilePath = GetExistingFilePath(text, true);
string fileName = lines[i].FileName;
if (existingFilePath != null)
text = existingFilePath;
GetImageForDocument(text, fileName, imageRecord.BatesNumber, batesNumber, i, ref offset, arrayList, i < lines.Count - 1, Convert.ToInt32(imageRecord.OriginalIndex), status, i == 0);
}
string arg = _fullTextStorageIsInSql ? "," : string.Empty;
if (_replaceFullText) {
if (!_fullTextStorageIsInSql)
_dataGridFileWriter.Write(batesNumber + "," + string.Empty + ",");
if (arrayList.Count == 0)
_bulkLoadFileWriter.Write($"{-1}{arg}");
else if (arrayList.Count > 0) {
_bulkLoadFileWriter.Write("{0}{1}", GetExtractedTextEncodings(arrayList).ToDelimitedString("|", "", "{0}"), arg);
}
StreamWriter streamWriter = _fullTextStorageIsInSql ? _bulkLoadFileWriter : _dataGridFileWriter;
IEnumerator enumerator2 = default(IEnumerator);
try {
enumerator2 = arrayList.GetEnumerator();
while (enumerator2.MoveNext()) {
string text2 = Conversions.ToString(enumerator2.Current);
Encoding encoding = _settings.FullTextEncoding;
Stream stream;
if (!SkipExtractedTextEncodingCheck) {
DeterminedEncodingStream determinedEncodingStream = Utility.DetectEncoding(text2, false, false);
stream = determinedEncodingStream.UnderlyingStream;
Encoding determinedEncoding = determinedEncodingStream.DeterminedEncoding;
if (determinedEncoding != null)
encoding = determinedEncoding;
} else
stream = new FileStream(text2, FileMode.Open, FileAccess.Read);
StreamReader streamReader = new StreamReader(stream, encoding, true);
streamWriter.Write(streamReader.ReadToEnd());
streamReader.Close();
try {
stream.Close();
} catch (Exception projectError) {
ProjectData.SetProjectError(projectError);
ProjectData.ClearProjectError();
}
streamReader = null;
}
} finally {
if (enumerator2 is IDisposable)
(enumerator2 as IDisposable).Dispose();
}
} else
_bulkLoadFileWriter.Write($"{-1}{arg}");
_bulkLoadFileWriter.Write("þþKþþ\r\n");
if (_replaceFullText && !_fullTextStorageIsInSql)
_dataGridFileWriter.Write("þþKþþ\r\n");
}
}
private List<int> GetExtractedTextEncodings(ArrayList textFileList)
{
List<int> list = new List<int>();
IEnumerator enumerator = default(IEnumerator);
try {
enumerator = textFileList.GetEnumerator();
while (enumerator.MoveNext()) {
string filename = Conversions.ToString(enumerator.Current);
Encoding encoding = _settings.FullTextEncoding;
if (!SkipExtractedTextEncodingCheck) {
Encoding determinedEncoding = Utility.DetectEncoding(filename, true, false).DeterminedEncoding;
if (determinedEncoding != null)
encoding = determinedEncoding;
}
if (!list.Contains(encoding.CodePage))
list.Add(encoding.CodePage);
}
return list;
} finally {
if (enumerator is IDisposable)
(enumerator as IDisposable).Dispose();
}
}
private void AutoNumberImages(List<ImageRecord> lines)
{
checked {
if (_autoNumberImages && lines.Count > 1) {
bool flag = true;
string batesNumber = lines[0].BatesNumber;
int num = lines.Count - 1;
for (int i = 0; i <= num; i++) {
flag = (flag && Operators.CompareString(batesNumber, lines[i].BatesNumber, false) == 0);
if (!flag)
return;
}
int num2 = lines.Count - 1;
for (int j = 1; j <= num2; j++) {
lines[j].BatesNumber = batesNumber + "_" + j.ToString().PadLeft(lines.Count.ToString().Length, '0');
}
}
}
}
private unsafe void GetImageForDocument(string imageFile, string originalFileName, string batesNumber, string documentIdentifier, int order, ref long offset, ArrayList fullTextFiles, bool writeLineTermination, int originalLineNumber, long status, bool isStartRecord)
{
checked {
ref long totalProcessed;
*(ref totalProcessed = ref _totalProcessed) = totalProcessed + 1;
string str = (!originalFileName.IsNullOrEmpty()) ? originalFileName : imageFile.Substring(imageFile.LastIndexOf("\\") + 1);
string text = imageFile.Substring(0, imageFile.LastIndexOf('.') + 1) + "txt";
string text2 = "";
string str2 = imageFile;
long value = 0;
ref int batchCount;
*(ref batchCount = ref _batchCount) = batchCount + 1;
if (status == 0) {
Guid guid;
if (_copyFilesToRepository && base.ShouldImport) {
base.ImportFilesCount++;
*(ref batchCount = ref _jobCompleteImageCount) = batchCount + 1;
UploadTapiBridge2 fileTapiBridge = base.FileTapiBridge;
guid = Guid.NewGuid();
text2 = fileTapiBridge.AddPath(imageFile, guid.ToString(), originalLineNumber);
str2 = base.FileTapiBridge.TargetPath.TrimEnd(new char[1] {
'\\'
}) + "\\" + base.FileTapiBridge.TargetFolderName + "\\" + text2;
} else {
WriteStatusLine(EventType2.Progress, $"""{batesNumber}""", originalLineNumber);
guid = Guid.NewGuid();
text2 = guid.ToString();
base.FileTapiProgressCount++;
}
if (GetFileExists(imageFile))
value = GetFileLength(imageFile, true);
if (_replaceFullText && GetFileExists(text) && fullTextFiles != null)
fullTextFiles.Add(text);
else if (_replaceFullText && !GetFileExists(text)) {
RaiseStatusEvent(EventType2.Warning, $"""{text}""", (long)Math.Round(unchecked((double)checked(_totalValidated + _totalProcessed) / 2)));
}
}
if (_replaceFullText && GetFileExists(text) && fullTextFiles != null)
offset += GetFileLength(text, true);
_bulkLoadFileWriter.Write(isStartRecord ? "1," : "0,");
_bulkLoadFileWriter.Write(Conversions.ToString(status) + ",");
_bulkLoadFileWriter.Write("0,");
_bulkLoadFileWriter.Write("0,");
_bulkLoadFileWriter.Write(Conversions.ToString(originalLineNumber) + ",");
_bulkLoadFileWriter.Write(documentIdentifier + ",");
_bulkLoadFileWriter.Write(batesNumber + ",");
_bulkLoadFileWriter.Write(text2 + ",");
_bulkLoadFileWriter.Write(str + ",");
_bulkLoadFileWriter.Write(Conversions.ToString(order) + ",");
_bulkLoadFileWriter.Write(Conversions.ToString(offset) + ",");
_bulkLoadFileWriter.Write(Conversions.ToString(value) + ",");
_bulkLoadFileWriter.Write(str2 + ",");
_bulkLoadFileWriter.Write(imageFile + ",");
_bulkLoadFileWriter.Write(",");
if (_replaceFullText && writeLineTermination)
_bulkLoadFileWriter.Write("-1,");
if (writeLineTermination)
_bulkLoadFileWriter.Write("þþKþþ\r\n");
}
}
private void RaiseFatalError(Exception ex)
{
FatalErrorEventEvent?.Invoke($"""{CurrentLineNumber}", ex);
}
private void RaiseStatusEvent(EventType2 et, string message, long progressLineNumber)
{
StatusMessageEvent?.Invoke(new StatusEventArgs(et, progressLineNumber, _recordCount, message, et == EventType2.Warning, base.CurrentStatisticsSnapshot, base.Statistics));
}
private void _processObserver_CancelImport(object sender, CancellationRequestEventArgs e)
{
if (Operators.CompareString(e.ProcessId.ToString(), _processID.ToString(), false) == 0) {
_cancelledByUser = e.RequestByUser;
StopImport(_cancelledByUser);
}
}
protected void OnStatusMessage(StatusEventArgs args)
{
StatusMessageEvent?.Invoke(args);
}
protected override void OnStopImport()
{
if (_imageReader != null)
_imageReader.Cancel();
RaiseStatusEvent(EventType2.Progress, $"""{base.TotalTransferredFilesCount}""", base.TotalTransferredFilesCount);
OnWriteStatusMessage(EventType2.Status, "Finalizing job…", 0, 0);
}
protected override void OnTapiClientChanged()
{
base.OnTapiClientChanged();
PublishUploadModeChangeEvent(_settings.CopyFilesToDocumentRepository);
}
protected override void OnWriteStatusMessage(EventType2 eventType, string message)
{
RaiseStatusEvent(EventType2.Status, message, CurrentLineNumber);
}
protected override void OnWriteStatusMessage(EventType2 eventType, string message, int progressLineNumber, int physicalLineNumber)
{
message = GetLineMessage(message, physicalLineNumber);
RaiseStatusEvent(eventType, message, progressLineNumber);
}
private void WriteStatusLine(EventType2 et, string line, int lineNumber)
{
int num = lineNumber;
if (num != 0)
num = num;
if (num < 0)
num = 0;
line = GetLineMessage(line, lineNumber);
OnStatusMessage(new StatusEventArgs(et, num, _recordCount, line, base.CurrentStatisticsSnapshot, base.Statistics));
}
protected override void OnWriteFatalError(Exception exception)
{
RaiseFatalError(exception);
}
private unsafe void RaiseReportError(Hashtable row)
{
ref int errorCount;
*(ref errorCount = ref _errorCount) = checked(errorCount + 1);
if (Operators.CompareString(_errorMessageFileLocation, "", false) == 0)
_errorMessageFileLocation = TempFileBuilder.GetTempFileName("rel-errors");
StreamWriter streamWriter = new StreamWriter(_errorMessageFileLocation, true, Encoding.Default);
if (_errorCount < MaxNumberOfErrorsInGrid)
ReportErrorEventEvent?.Invoke(row);
else if (_errorCount == MaxNumberOfErrorsInGrid) {
Hashtable hashtable = new Hashtable();
hashtable.Add("Message", "Maximum number of errors for display reached. Export errors to view full list.");
ReportErrorEventEvent?.Invoke(hashtable);
}
streamWriter.WriteLine(string.Format("{0},{1},{2},{3}", CSVFormat(row["Line Number"].ToString()), CSVFormat(row["DocumentID"].ToString()), CSVFormat(row["FileID"].ToString()), CSVFormat(row["Message"].ToString())));
streamWriter.Close();
}
private string CSVFormat(string fieldValue)
{
return "\"" + fieldValue.Replace("\"", "\"\"") + "\"";
}
private void IoWarningHandler(object sender, IoWarningEventArgs e)
{
IoWarningEventArgs args = new IoWarningEventArgs(e.Message, e.CurrentLineNumber);
PublishIoWarningEvent(args);
}
private void ManagePrePushErrors()
{
if (_prePushErrors.Any()) {
InitializeErrorFiles();
using (StreamWriter streamWriter = new StreamWriter(_errorRowsFileLocation, true, Encoding.Default)) {
foreach (Tuple<ImageRecord, string> prePushError in _prePushErrors) {
ImageRecord item = prePushError.Item1;
string item2 = prePushError.Item2;
string[] line = new string[4] {
item.OriginalIndex.ToString(),
item.BatesNumber,
item.FileName,
item2
};
ProcessError(line);
string text = item.IsNewDoc ? "Y" : string.Empty;
streamWriter.WriteLine($"{item.BatesNumber}""{RunId}""{item.FileLocation}""{text}""");
}
}
_prePushErrors = new List<Tuple<ImageRecord, string>>();
}
}
private void ManageErrors()
{
if (_bulkImportManager.ImageRunHasErrors(_caseInfo.ArtifactID, _runId)) {
InitializeErrorFiles();
StreamWriter streamWriter = null;
StreamReader streamReader = null;
GenericCsvReader2 genericCsvReader = null;
try {
global::Relativity.DataExchange.Service.ErrorFileKey errorFileKey = _bulkImportManager.GenerateImageErrorFiles(_caseInfo.ArtifactID, _runId, true, _keyFieldDto.ArtifactID);
RaiseStatusEvent(EventType2.Status, "Retrieving errors from server", CurrentLineNumber);
ErrorFileService errorFileService = new ErrorFileService((NetworkCredential)_bulkImportManager.Credentials, _caseInfo.DownloadHandlerURL, _bulkImportManager.CookieContainer, GetCorrelationId);
string tempFileName = TempFileBuilder.GetTempFileName("rel-errors");
genericCsvReader = AttemptErrorFileDownload(errorFileService, tempFileName, errorFileKey.LogKey, _caseInfo);
if (genericCsvReader == null)
FatalErrorEventEvent?.Invoke("There was an error while attempting to retrieve the errors from the server.", new Exception("There was an error while attempting to retrieve the errors from the server."));
else {
genericCsvReader.Context.IoWarningEvent += IoWarningHandler;
for (string[] array = genericCsvReader.ReadLine(); array != null; array = genericCsvReader.ReadLine()) {
ProcessError(array);
}
genericCsvReader.Close();
string tempFileName2 = TempFileBuilder.GetTempFileName("rel-errors");
errorFileService.DownloadErrorFile(tempFileName2, errorFileKey.OpticonKey, _caseInfo);
streamWriter = new StreamWriter(_errorRowsFileLocation, true, Encoding.Default);
streamReader = new StreamReader(tempFileName2, Encoding.Default);
for (int num = streamReader.Read(); num != -1; num = streamReader.Read()) {
streamWriter.Write(Strings.ChrW(num));
}
streamWriter.Close();
streamReader.Close();
global::Relativity.DataExchange.Io.FileSystem.Instance.File.Delete(tempFileName2);
genericCsvReader.Context.IoWarningEvent -= IoWarningHandler;
}
errorFileKey = null;
} catch (Exception ex) {
ProjectData.SetProjectError(ex);
Exception exception = ex;
LogWarning(exception, "Failed to manage the image import errors.");
try {
genericCsvReader?.Close();
genericCsvReader.Context.IoWarningEvent -= IoWarningHandler;
} catch (Exception ex2) {
ProjectData.SetProjectError(ex2);
Exception exception2 = ex2;
LogWarning(exception2, "Failed to close the image import CSV stream.");
ProjectData.ClearProjectError();
}
try {
streamWriter?.Close();
} catch (Exception ex3) {
ProjectData.SetProjectError(ex3);
Exception exception3 = ex3;
LogWarning(exception3, "Failed to close the image import error lines stream.");
ProjectData.ClearProjectError();
}
try {
streamReader?.Close();
} catch (Exception ex4) {
ProjectData.SetProjectError(ex4);
Exception exception4 = ex4;
LogWarning(exception4, "Failed to close the downloaded image import errors stream.");
ProjectData.ClearProjectError();
}
ProjectData.ClearProjectError();
} finally {
_verboseErrorCollection.Clear();
}
}
}
private void InitializeErrorFiles()
{
if (Operators.CompareString(_errorMessageFileLocation, "", false) == 0)
_errorMessageFileLocation = TempFileBuilder.GetTempFileName("rel-errors");
if (Operators.CompareString(_errorRowsFileLocation, "", false) == 0)
_errorRowsFileLocation = TempFileBuilder.GetTempFileName("rel-errors");
}
private void ProcessError(string[] line)
{
long num = long.Parse(line[0]);
Hashtable hashtable = new Hashtable();
checked {
hashtable.Add("Line Number", (int)num);
hashtable.Add("DocumentID", line[1]);
hashtable.Add("FileID", line[2]);
string text = line[3];
if (_verboseErrorCollection.get_ContainsLine(num)) {
StringBuilder stringBuilder = new StringBuilder();
foreach (string item in _verboseErrorCollection[num]) {
stringBuilder.Append(ImportStatusHelper.ConvertToMessageLineInCell(item));
}
text = stringBuilder.ToString().TrimEnd(new char[1] {
'\n'
});
}
hashtable.Add("Message", text);
RaiseReportError(hashtable);
long num2 = num + base.ImportFilesCount;
StatusMessageEvent?.Invoke(new StatusEventArgs(EventType2.Error, num2 - 1, _recordCount, "[Line " + line[0] + "]" + text, null, base.Statistics));
}
}
private GenericCsvReader2 AttemptErrorFileDownload(ErrorFileService errorFileService, string errorFileOutputPath, string logKey, CaseInfo caseInfo)
{
int num = 3;
GenericCsvReader2 genericCsvReader = null;
while (num > 0) {
errorFileService.DownloadErrorFile(errorFileOutputPath, logKey, caseInfo, false);
genericCsvReader = new GenericCsvReader2(errorFileOutputPath, true);
if (genericCsvReader.Peek() != -1)
break;
num = checked(num - 1);
genericCsvReader.Close();
genericCsvReader = null;
}
errorFileService.RemoveErrorFile(logKey, caseInfo);
return genericCsvReader;
}
private void _processContext_ExportServerErrorsEvent(object sender, ExportErrorEventArgs e)
{
string text = _filePath;
string str;
if (text.IndexOf(".") != -1) {
str = text.Substring(text.LastIndexOf("."));
text = text.Substring(0, text.LastIndexOf("."));
} else
str = ".opt";
if (text.IndexOf("\\") != -1)
text = text.Substring(checked(text.LastIndexOf("\\") + 1));
string str2 = e.Path + text;
DateTime now = DateTime.Now;
string destFileName = str2 + "_ErrorLines_" + Conversions.ToString(now.Ticks) + str;
string destFileName2 = str2 + "_ErrorReport_" + Conversions.ToString(now.Ticks) + ".csv";
CopyFile(_errorRowsFileLocation, destFileName);
CopyFile(_errorMessageFileLocation, destFileName2);
}
private void _processContext_ExportErrorFileEvent(object sender, ExportErrorEventArgs e)
{
try {
if (GetFileExists(_errorRowsFileLocation))
CopyFile(_errorRowsFileLocation, e.Path, true);
} catch (Exception ex) {
ProjectData.SetProjectError(ex);
Exception exception = ex;
LogWarning(exception, "Failed to copy the image import error rows file. Going to retry the copy...");
if (GetFileExists(_errorRowsFileLocation)) {
CopyFile(_errorRowsFileLocation, e.Path, true);
LogInformation("Successfully copied the image import error rows file on retry.");
}
ProjectData.ClearProjectError();
}
}
private void _processContext_ExportErrorReportEvent(object sender, ExportErrorEventArgs e)
{
if (!string.IsNullOrEmpty(_errorMessageFileLocation))
try {
if (GetFileExists(_errorMessageFileLocation))
CopyFile(_errorMessageFileLocation, e.Path, true);
} catch (Exception ex) {
ProjectData.SetProjectError(ex);
Exception exception = ex;
LogWarning(exception, "Failed to copy the image import error location file. Going to retry the copy...");
if (GetFileExists(_errorMessageFileLocation)) {
CopyFile(_errorMessageFileLocation, e.Path, true);
LogInformation("Successfully copied the image import error location file.");
}
ProjectData.ClearProjectError();
}
else
File.CreateText(e.Path).Close();
}
private void _processContext_ParentFormClosingEvent(object sender, ParentFormClosingEventArgs e)
{
if (Operators.CompareString(e.ProcessId.ToString(), _processID.ToString(), false) == 0)
CleanupTempTables();
}
private void CleanupTempTables()
{
if (_runId != null && Operators.CompareString(_runId, "", false) != 0)
try {
_bulkImportManager.DisposeTempTables(_caseInfo.ArtifactID, _runId);
} catch (Exception ex) {
ProjectData.SetProjectError(ex);
Exception exception = ex;
LogWarning(exception, "Failed to drop the {RunId} SQL temp tables.", _runId);
ProjectData.ClearProjectError();
}
}
private string GetCorrelationId()
{
if (string.IsNullOrEmpty(_runId)) {
if (string.IsNullOrEmpty(_localRunId))
return CorrelationIdFunc?.Invoke();
return _localRunId;
}
return _runId;
}
}
}