BulkLoadFileImporter
using kCura.EDDS.WebAPI.BulkImportManagerBase;
using kCura.EDDS.WebAPI.DocumentManagerBase;
using kCura.WinEDDS.Api;
using kCura.WinEDDS.CodeValidator;
using kCura.WinEDDS.Exceptions;
using kCura.WinEDDS.Service;
using kCura.WinEDDS.Service.Replacement;
using Microsoft.VisualBasic;
using Microsoft.VisualBasic.CompilerServices;
using Monitoring;
using Relativity.DataExchange;
using Relativity.DataExchange.Data;
using Relativity.DataExchange.Io;
using Relativity.DataExchange.Logger;
using Relativity.DataExchange.Logging;
using Relativity.DataExchange.Process;
using Relativity.DataExchange.Service;
using Relativity.DataExchange.Transfer;
using Relativity.Logging;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.Data;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net;
using System.Runtime.CompilerServices;
using System.Runtime.Serialization;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Web;
namespace kCura.WinEDDS
{
public class BulkLoadFileImporter : LoadFileBase, IImportJob, IDisposable
{
public delegate void FatalErrorEventEventHandler (string message, Exception ex, string runID);
public delegate void StatusMessageEventHandler (StatusEventArgs args);
public delegate void EndFileImportEventHandler (string runID);
public delegate void StartFileImportEventHandler ();
public delegate void ReportErrorEventEventHandler (IDictionary row);
public delegate void DataSourcePrepEventEventHandler (DataSourcePrepEventArgs e);
public delegate void FieldMappedEventHandler (string sourceField, string workspaceField);
[Serializable]
public class IdentityValueNotSetException : ImporterException
{
public IdentityValueNotSetException()
: base("Identity value not set")
{
}
protected IdentityValueNotSetException(SerializationInfo info, StreamingContext context)
: base(info, context)
{
}
}
[Serializable]
public class ExtractedTextFileNotFoundException : ImporterException
{
public ExtractedTextFileNotFoundException()
: base("Error occurred when importing the document. Extracted text is missing.")
{
}
protected ExtractedTextFileNotFoundException(SerializationInfo info, StreamingContext context)
: base(info, context)
{
}
}
[Serializable]
public new class ExtractedTextTooLargeException : ImporterException
{
public ExtractedTextTooLargeException()
: base("Error occurred when importing the document. Extracted text is greater than 2 GB.")
{
}
protected ExtractedTextTooLargeException(SerializationInfo info, StreamingContext context)
: base(info, context)
{
}
}
[Serializable]
[CompilerGenerated]
internal sealed class _Closure$__
{
public static readonly _Closure$__ $I;
public static Func<ArtifactField, bool> $I136-4;
public static Func<string, bool> $I138-0;
public static Func<Exception, bool> $I152-0;
static _Closure$__()
{
$I = new _Closure$__();
}
internal bool _Lambda$__136-4(ArtifactField x)
{
return Operators.CompareString(x.DisplayName, "DataGridID", false) == 0;
}
internal bool _Lambda$__138-0(string path)
{
return path.Length > 255;
}
internal bool _Lambda$__152-0(Exception e)
{
throw e;
}
}
[CompilerGenerated]
internal sealed class _Closure$__136-0
{
public int $VB$Local_currentRetryAttempt;
public int $VB$Local_maxRetryAttempts;
public _Closure$__136-1 $VB$NonLocal_$VB$Closure_2;
internal TimeSpan _Lambda$__1(int count)
{
$VB$Local_currentRetryAttempt = count;
if (count > 1)
$VB$NonLocal_$VB$Closure_2.$VB$Local_fileTypeIdentifier.Reinitialize();
return TimeSpan.FromSeconds((double)$VB$NonLocal_$VB$Closure_2.$VB$Me.WaitTimeBetweenRetryAttempts);
}
internal void _Lambda$__2(Exception exception, TimeSpan span)
{
$VB$NonLocal_$VB$Closure_2.$VB$Me.PublishIoRetryMessage(exception, span, $VB$Local_currentRetryAttempt, $VB$Local_maxRetryAttempts);
}
}
[CompilerGenerated]
internal sealed class _Closure$__136-1
{
public IFileTypeIdentifier $VB$Local_fileTypeIdentifier;
public string $VB$Local_fullFilePath;
public BulkLoadFileImporter $VB$Me;
internal IFileTypeInfo _Lambda$__R1(CancellationToken a0)
{
return _Lambda$__3();
}
internal IFileTypeInfo _Lambda$__3()
{
return $VB$Local_fileTypeIdentifier.Identify($VB$Local_fullFilePath);
}
}
[CompilerGenerated]
internal sealed class _Closure$__151-0
{
public NativeLoadInfo $VB$Local_settings;
public BulkLoadFileImporter $VB$Me;
internal void _Lambda$__0()
{
DateTime now = DateTime.Now;
MassImportResults massImportResults = $VB$Me.BulkImport($VB$Local_settings, $VB$Me._fullTextColumnMapsToFileLocation);
$VB$Me.Statistics.ProcessMassImportResults(massImportResults);
TimeSpan timeSpan = DateTime.Now - now;
ImportStatistics statistics;
(statistics = $VB$Me.Statistics).MassImportDuration = statistics.MassImportDuration + timeSpan;
(statistics = $VB$Me.Statistics).BatchCount = checked(statistics.BatchCount + 1);
$VB$Me.Logger.LogInformation("Duration of mass import processing: {durationInMilliseconds}, batch: {numberOfBatch}", new object[2] {
timeSpan.TotalMilliseconds,
$VB$Me.Statistics.BatchCount
});
$VB$Me.UpdateStatisticsSnapshot(DateTime.Now, false);
$VB$Me.ManageErrors($VB$Me._artifactTypeID);
BatchInformation batchInformation = new BatchInformation {
OrdinalNumber = $VB$Me.Statistics.BatchCount,
NumberOfRecords = massImportResults.FilesProcessed,
MassImportDuration = timeSpan
};
$VB$Me.OnBatchCompleted(batchInformation);
}
}
[CompilerGenerated]
internal sealed class _Closure$__162-0
{
public int $VB$Local_currentRetryAttempt;
public int $VB$Local_maxRetryAttempts;
public _Closure$__162-1 $VB$NonLocal_$VB$Closure_2;
public _Closure$__162-0(_Closure$__162-0 arg0)
{
if (arg0 != null) {
$VB$Local_currentRetryAttempt = arg0.$VB$Local_currentRetryAttempt;
$VB$Local_maxRetryAttempts = arg0.$VB$Local_maxRetryAttempts;
}
}
internal TimeSpan _Lambda$__0(int count)
{
$VB$Local_currentRetryAttempt = count;
return TimeSpan.FromSeconds((double)$VB$NonLocal_$VB$Closure_2.$VB$Me.WaitTimeBetweenRetryAttempts);
}
internal void _Lambda$__1(Exception exception, TimeSpan span)
{
$VB$NonLocal_$VB$Closure_2.$VB$Me.PublishIoRetryMessage(exception, span, $VB$Local_currentRetryAttempt, $VB$Local_maxRetryAttempts);
}
}
[CompilerGenerated]
internal sealed class _Closure$__162-1
{
public Encoding $VB$Local_extractedTextEncoding;
public ArtifactField $VB$Local_field;
public IStreamWriter $VB$Local_outputWriter;
public BulkLoadFileImporter $VB$Me;
public _Closure$__162-1(_Closure$__162-1 arg0)
{
if (arg0 != null) {
$VB$Local_extractedTextEncoding = arg0.$VB$Local_extractedTextEncoding;
$VB$Local_field = arg0.$VB$Local_field;
$VB$Local_outputWriter = arg0.$VB$Local_outputWriter;
}
}
internal Encoding _Lambda$__2(CancellationToken token)
{
Encoding encoding = $VB$Local_extractedTextEncoding;
if ($VB$Me.GetFileLength($VB$Local_field.ValueAsString, false) > $VB$Me.GetMaxExtractedTextLength(encoding))
throw new ExtractedTextTooLargeException();
if (!$VB$Me.LoadImportedFullTextFromServer) {
Stream stream;
if (!$VB$Me.SkipExtractedTextEncodingCheck) {
DeterminedEncodingStream determinedEncodingStream = Utility.DetectEncoding($VB$Local_field.ValueAsString, false);
stream = determinedEncodingStream.UnderlyingStream;
Encoding determinedEncoding = determinedEncodingStream.DeterminedEncoding;
if (determinedEncoding != null)
encoding = determinedEncoding;
} else
stream = new FileStream($VB$Local_field.ValueAsString, FileMode.Open, FileAccess.Read);
StreamReader streamReader = new StreamReader(stream, encoding);
char[] buffer = new char[40001];
int num;
do {
num = streamReader.ReadBlock(buffer, 0, 40000);
if (num > 0) {
$VB$Local_outputWriter.Write(buffer, 0, num);
$VB$Local_outputWriter.Flush();
}
} while (num != 0);
streamReader.Close();
try {
stream.Close();
return encoding;
} catch (Exception projectError) {
ProjectData.SetProjectError(projectError);
ProjectData.ClearProjectError();
return encoding;
}
}
if (!$VB$Me.SkipExtractedTextEncodingCheck) {
DeterminedEncodingStream determinedEncodingStream2 = Utility.DetectEncoding($VB$Local_field.ValueAsString, false);
Stream stream = determinedEncodingStream2.UnderlyingStream;
Field field = $VB$Me.get_FullTextField($VB$Me._settings.ArtifactTypeID);
Encoding objA = (field != null && field.UseUnicodeEncoding) ? Encoding.Unicode : null;
Encoding determinedEncoding2 = determinedEncodingStream2.DeterminedEncoding;
if (!object.Equals(objA, determinedEncoding2))
$VB$Me.WriteWarning("The extracted text file's encoding was not detected to be the same as the extracted text field. The imported data may be incorrectly encoded.");
if (determinedEncoding2 != null)
encoding = determinedEncoding2;
try {
stream.Close();
} catch (Exception projectError2) {
ProjectData.SetProjectError(projectError2);
ProjectData.ClearProjectError();
}
}
$VB$Local_outputWriter.Write(RuntimeHelpers.GetObjectValue($VB$Local_field.Value));
return encoding;
}
}
public static readonly string RestartTimeEventMsg = "Reset time for import rolling average";
public static readonly string CancelEventMsg = "cancel import";
private const int _COPY_TEXT_FILE_BUFFER_SIZE = 40000;
private const int _UNKNOWN_PARENT_FOLDER_ID = -9;
private const int _LENGTH_OF_FOLDER_ALLOWED = 255;
public const string DATA_GRID_ID_FIELD_NAME = "DataGridID";
public const string ERROR_MESSAGE_FOLDER_NAME_TOO_LONG = "Error occurred when importing the document. The folder name is longer than 255 characters.";
private readonly bool _copyFileToRepository;
private readonly CaseInfo _caseInfo;
private readonly bool _usePipeliningForNativeAndObjectImports;
private readonly bool _createFoldersInWebApi;
private readonly bool _createErrorForEmptyNativeFile;
protected ImportOverwriteType Overwrite;
protected IAuditManager AuditManager;
protected IRelativityManager RelativityManager;
protected long RecordCount;
private Field[] _allFields;
private Field[] _fieldsForCreate;
private Dictionary<string, int> _processedKeyFieldValues;
[CompilerGenerated]
[AccessedThroughProperty("Context")]
private ProcessContext _Context;
protected int Offset;
protected bool FirstTimeThrough;
private int? _importBatchSize;
private int? _jobCompleteBatchSize;
private int? _importBatchVolume;
private int? _minimumBatchSize;
protected int DestinationFolderColumnIndex;
protected FolderCache FolderCache;
private string _defaultDestinationFolderPath;
private HybridDictionary _oixFileLookup;
private int[] _fieldArtifactIds;
protected OutputFileWriter OutputFileWriter;
protected int OverlayArtifactId;
private string _runId;
private long _lastRunMetadataImport;
private ITimeKeeperManager _timekeeper;
private string _filePath;
protected int _batchCounter;
private int _jobCompleteNativeCount;
private int _jobCompleteMetadataCount;
private string _errorLinesFileLocation;
public int MaxNumberOfErrorsInGrid;
private int _errorCount;
private ErrorMessageWriter<ErrorBeforeMassImportArgs> prePushErrorWriter;
private ErrorMessageWriter<ErrorDuringMassImportArgs> errorMessageFileWriter;
private Guid _processId;
private int? _parentArtifactTypeId;
private ArrayList _unmappedRelationalFields;
private bool _cancelledByUser;
protected string BulkLoadFileFieldDelimiter;
[CompilerGenerated]
private bool _LinkDataGridRecords;
[CompilerGenerated]
private bool _DisableNativeValidation;
[CompilerGenerated]
private bool _DisableUserSecurityCheck;
[CompilerGenerated]
private ImportAuditLevel _AuditLevel;
[CompilerGenerated]
private List<int> _BatchSizeHistoryList;
private Task _task;
[CompilerGenerated]
private FatalErrorEventEventHandler FatalErrorEventEvent;
[CompilerGenerated]
private StatusMessageEventHandler StatusMessageEvent;
[CompilerGenerated]
private EndFileImportEventHandler EndFileImportEvent;
[CompilerGenerated]
private StartFileImportEventHandler StartFileImportEvent;
[CompilerGenerated]
private ReportErrorEventEventHandler ReportErrorEventEvent;
[CompilerGenerated]
private DataSourcePrepEventEventHandler DataSourcePrepEventEvent;
[CompilerGenerated]
private FieldMappedEventHandler FieldMappedEvent;
protected virtual ProcessContext Context {
[CompilerGenerated]
get {
return _Context;
}
[MethodImpl(MethodImplOptions.Synchronized)]
[CompilerGenerated]
set {
EventHandler<CancellationRequestEventArgs> value2 = _processContext_HaltProcessEvent;
EventHandler<ExportErrorEventArgs> value3 = _processContext_ExportServerErrors;
EventHandler<ExportErrorEventArgs> value4 = _processContext_ExportErrorReportEvent;
EventHandler<ExportErrorEventArgs> value5 = _processContext_ExportErrorFileEvent;
EventHandler<ParentFormClosingEventArgs> value6 = _processContext_ParentFormClosingEvent;
ProcessContext context = _Context;
if (context != null) {
context.CancellationRequest -= value2;
context.ExportServerErrors -= value3;
context.ExportErrorReport -= value4;
context.ExportErrorFile -= value5;
context.ParentFormClosing -= value6;
}
_Context = value;
context = _Context;
if (context != null) {
context.CancellationRequest += value2;
context.ExportServerErrors += value3;
context.ExportErrorReport += value4;
context.ExportErrorFile += value5;
context.ParentFormClosing += value6;
}
}
}
protected bool LinkDataGridRecords { get; set; }
public ITimeKeeperManager Timekeeper {
get {
if (_timekeeper == null)
_timekeeper = new DefaultTimeKeeperManager();
return _timekeeper;
}
set {
_timekeeper = value;
}
}
internal long TotalRecords {
get {
if (RecordCount <= 0)
RecordCount = _artifactReader.CountRecords().GetValueOrDefault();
return RecordCount;
}
}
internal long CompletedRecords => base.TotalTransferredFilesCount;
internal bool IsCancelledByUser => _cancelledByUser;
public bool DisableNativeValidation { get; set; }
public bool DisableUserSecurityCheck { get; set; }
public ImportAuditLevel AuditLevel { get; set; }
public List<int> BatchSizeHistoryList { get; }
protected virtual int NumberOfRetries => AppSettings.Instance.IoErrorNumberOfRetries;
protected virtual int WaitTimeBetweenRetryAttempts => AppSettings.Instance.IoErrorWaitTimeInSeconds;
public Field[] AllFields {
get {
if (_allFields == null)
_allFields = _fieldQuery.RetrieveAllAsArray(_caseArtifactID, artifactTypeID, true);
Field[] allFields = _allFields;
foreach (Field field in allFields) {
field.Value = null;
field.FieldCategory = (kCura.EDDS.WebAPI.DocumentManagerBase.FieldCategory)field.FieldCategoryID;
}
return _allFields;
}
}
public Field[] DocumentFieldsForCreate {
get {
if (_fieldsForCreate == null) {
ArrayList arrayList = new ArrayList();
Field[] array = this.get_AllFields(10);
foreach (Field field in array) {
if (Array.IndexOf(_fieldArtifactIds, field.ArtifactID) != -1)
arrayList.Add(field);
}
_fieldsForCreate = (Field[])arrayList.ToArray(typeof(Field));
}
return _fieldsForCreate;
}
}
public Field FileInfoField {
get {
Field result = new Field();
Field[] array = this.get_AllFields(artifactTypeID);
foreach (Field field in array) {
if (field.FieldCategoryID == 10)
result = field;
}
return result;
}
}
public Field FullTextField {
get {
Field[] array = this.get_AllFields(artifactTypeID);
foreach (Field field in array) {
if (field.FieldCategory == kCura.EDDS.WebAPI.DocumentManagerBase.FieldCategory.FullText)
return field;
}
return null;
}
}
public virtual bool HasErrors => _errorCount > 0;
public TapiClient UploadConnection => base.FileTapiClient;
public int FoldersCreated => _folderManager.CreationCount;
public int CodesCreated {
get {
if (MulticodeMatrix != null) {
IEnumerator enumerator = default(IEnumerator);
int num = default(int);
try {
enumerator = MulticodeMatrix.Values.GetEnumerator();
while (enumerator.MoveNext()) {
NestedArtifactCache nestedArtifactCache = (NestedArtifactCache)enumerator.Current;
num = checked(num + nestedArtifactCache.CreationCount);
}
} finally {
if (enumerator is IDisposable)
(enumerator as IDisposable).Dispose();
}
return num;
}
return 0;
}
}
public ArrayList UnmappedRelationalFields {
get {
if (_unmappedRelationalFields == null) {
ArrayList arrayList = new ArrayList();
IEnumerator enumerator = default(IEnumerator);
try {
enumerator = _fieldMap.GetEnumerator();
while (enumerator.MoveNext()) {
LoadFileFieldMap.LoadFileFieldMapItem loadFileFieldMapItem = (LoadFileFieldMap.LoadFileFieldMapItem)enumerator.Current;
if (loadFileFieldMapItem.DocumentField != null && loadFileFieldMapItem.DocumentField.FieldCategory == global::Relativity.DataExchange.Service.FieldCategory.Relational) {
int? nullable = (int?)loadFileFieldMapItem.DocumentField.ImportBehavior;
if ((nullable.HasValue ? new bool?(nullable.GetValueOrDefault() == 1) : null).GetValueOrDefault())
arrayList.Add(loadFileFieldMapItem.DocumentField.FieldID);
}
}
} finally {
if (enumerator is IDisposable)
(enumerator as IDisposable).Dispose();
}
_unmappedRelationalFields = new ArrayList();
Field[] array = this.get_AllFields(_artifactTypeID);
foreach (Field field in array) {
if ((field.FieldCategory == kCura.EDDS.WebAPI.DocumentManagerBase.FieldCategory.Relational) & !arrayList.Contains(field.ArtifactID)) {
int? nullable = (int?)field.ImportBehavior;
if ((nullable.HasValue ? new bool?(nullable.GetValueOrDefault() == 1) : null).GetValueOrDefault())
_unmappedRelationalFields.Add(field);
}
}
}
return _unmappedRelationalFields;
}
}
public string RunId => _runId;
protected virtual int ParentArtifactTypeID {
get {
if (!_parentArtifactTypeId.HasValue) {
IObjectTypeManager objectTypeManager = ManagerFactory.CreateObjectTypeManager(_settings.Credentials, _settings.CookieContainer, GetCorrelationId);
_parentArtifactTypeId = Conversions.ToInteger(objectTypeManager.RetrieveParentArtifactTypeID(_settings.CaseInfo.ArtifactID, _settings.ArtifactTypeID).Tables[0].Rows[0]["ParentArtifactTypeID"]);
}
return _parentArtifactTypeId.Value;
}
}
protected virtual int MinimumBatchSize {
get {
if (!_minimumBatchSize.HasValue)
_minimumBatchSize = AppSettings.Instance.MinBatchSize;
return _minimumBatchSize.Value;
}
set {
_minimumBatchSize = value;
}
}
protected int ImportBatchSize {
get {
if (!_importBatchSize.HasValue)
_importBatchSize = AppSettings.Instance.ImportBatchSize;
return _importBatchSize.Value;
}
set {
_importBatchSize = ((value > MinimumBatchSize) ? value : MinimumBatchSize);
}
}
protected int JobCompleteBatchSize {
get {
if (!_jobCompleteBatchSize.HasValue)
_jobCompleteBatchSize = AppSettings.Instance.JobCompleteBatchSize;
return _jobCompleteBatchSize.Value;
}
set {
_jobCompleteBatchSize = ((value > MinimumBatchSize) ? value : MinimumBatchSize);
}
}
protected int ImportBatchVolume {
get {
if (!_importBatchVolume.HasValue)
_importBatchVolume = AppSettings.Instance.ImportBatchMaxVolume;
return _importBatchVolume.Value;
}
set {
_importBatchVolume = value;
}
}
protected virtual bool BatchResizeEnabled => AppSettings.Instance.DynamicBatchResizingOn;
protected override bool UseTimeZoneOffset => true;
protected override IArtifactReader _artifactReader {
[CompilerGenerated]
get {
return base._artifactReader;
}
[MethodImpl(MethodImplOptions.Synchronized)]
[CompilerGenerated]
set {
IArtifactReader.DataSourcePrepEventHandler obj = _artifactReader_DataSourcePrep;
IArtifactReader.StatusMessageEventHandler obj2 = _artifactReader_StatusMessage;
IArtifactReader.FieldMappedEventHandler obj3 = _artifactReader_FieldMapped;
IArtifactReader artifactReader = base._artifactReader;
if (artifactReader != null) {
artifactReader.DataSourcePrep -= obj;
artifactReader.StatusMessage -= obj2;
artifactReader.FieldMapped -= obj3;
}
base._artifactReader = value;
artifactReader = base._artifactReader;
if (artifactReader != null) {
artifactReader.DataSourcePrep += obj;
artifactReader.StatusMessage += obj2;
artifactReader.FieldMapped += obj3;
}
}
}
public event FatalErrorEventEventHandler FatalErrorEvent {
[CompilerGenerated]
add {
FatalErrorEventEventHandler fatalErrorEventEventHandler = FatalErrorEventEvent;
FatalErrorEventEventHandler fatalErrorEventEventHandler2;
do {
fatalErrorEventEventHandler2 = fatalErrorEventEventHandler;
FatalErrorEventEventHandler value2 = (FatalErrorEventEventHandler)Delegate.Combine(fatalErrorEventEventHandler2, value);
fatalErrorEventEventHandler = Interlocked.CompareExchange(ref FatalErrorEventEvent, value2, fatalErrorEventEventHandler2);
} while ((object)fatalErrorEventEventHandler != fatalErrorEventEventHandler2);
}
[CompilerGenerated]
remove {
FatalErrorEventEventHandler fatalErrorEventEventHandler = FatalErrorEventEvent;
FatalErrorEventEventHandler fatalErrorEventEventHandler2;
do {
fatalErrorEventEventHandler2 = fatalErrorEventEventHandler;
FatalErrorEventEventHandler value2 = (FatalErrorEventEventHandler)Delegate.Remove(fatalErrorEventEventHandler2, value);
fatalErrorEventEventHandler = Interlocked.CompareExchange(ref FatalErrorEventEvent, value2, fatalErrorEventEventHandler2);
} while ((object)fatalErrorEventEventHandler != fatalErrorEventEventHandler2);
}
}
public event StatusMessageEventHandler StatusMessage {
[CompilerGenerated]
add {
StatusMessageEventHandler statusMessageEventHandler = StatusMessageEvent;
StatusMessageEventHandler statusMessageEventHandler2;
do {
statusMessageEventHandler2 = statusMessageEventHandler;
StatusMessageEventHandler value2 = (StatusMessageEventHandler)Delegate.Combine(statusMessageEventHandler2, value);
statusMessageEventHandler = Interlocked.CompareExchange(ref StatusMessageEvent, value2, statusMessageEventHandler2);
} while ((object)statusMessageEventHandler != statusMessageEventHandler2);
}
[CompilerGenerated]
remove {
StatusMessageEventHandler statusMessageEventHandler = StatusMessageEvent;
StatusMessageEventHandler statusMessageEventHandler2;
do {
statusMessageEventHandler2 = statusMessageEventHandler;
StatusMessageEventHandler value2 = (StatusMessageEventHandler)Delegate.Remove(statusMessageEventHandler2, value);
statusMessageEventHandler = Interlocked.CompareExchange(ref StatusMessageEvent, value2, statusMessageEventHandler2);
} while ((object)statusMessageEventHandler != statusMessageEventHandler2);
}
}
public event EndFileImportEventHandler EndFileImport {
[CompilerGenerated]
add {
EndFileImportEventHandler endFileImportEventHandler = EndFileImportEvent;
EndFileImportEventHandler endFileImportEventHandler2;
do {
endFileImportEventHandler2 = endFileImportEventHandler;
EndFileImportEventHandler value2 = (EndFileImportEventHandler)Delegate.Combine(endFileImportEventHandler2, value);
endFileImportEventHandler = Interlocked.CompareExchange(ref EndFileImportEvent, value2, endFileImportEventHandler2);
} while ((object)endFileImportEventHandler != endFileImportEventHandler2);
}
[CompilerGenerated]
remove {
EndFileImportEventHandler endFileImportEventHandler = EndFileImportEvent;
EndFileImportEventHandler endFileImportEventHandler2;
do {
endFileImportEventHandler2 = endFileImportEventHandler;
EndFileImportEventHandler value2 = (EndFileImportEventHandler)Delegate.Remove(endFileImportEventHandler2, value);
endFileImportEventHandler = Interlocked.CompareExchange(ref EndFileImportEvent, value2, endFileImportEventHandler2);
} while ((object)endFileImportEventHandler != endFileImportEventHandler2);
}
}
public event StartFileImportEventHandler StartFileImport {
[CompilerGenerated]
add {
StartFileImportEventHandler startFileImportEventHandler = StartFileImportEvent;
StartFileImportEventHandler startFileImportEventHandler2;
do {
startFileImportEventHandler2 = startFileImportEventHandler;
StartFileImportEventHandler value2 = (StartFileImportEventHandler)Delegate.Combine(startFileImportEventHandler2, value);
startFileImportEventHandler = Interlocked.CompareExchange(ref StartFileImportEvent, value2, startFileImportEventHandler2);
} while ((object)startFileImportEventHandler != startFileImportEventHandler2);
}
[CompilerGenerated]
remove {
StartFileImportEventHandler startFileImportEventHandler = StartFileImportEvent;
StartFileImportEventHandler startFileImportEventHandler2;
do {
startFileImportEventHandler2 = startFileImportEventHandler;
StartFileImportEventHandler value2 = (StartFileImportEventHandler)Delegate.Remove(startFileImportEventHandler2, value);
startFileImportEventHandler = Interlocked.CompareExchange(ref StartFileImportEvent, value2, startFileImportEventHandler2);
} while ((object)startFileImportEventHandler != startFileImportEventHandler2);
}
}
public event ReportErrorEventEventHandler ReportErrorEvent {
[CompilerGenerated]
add {
ReportErrorEventEventHandler reportErrorEventEventHandler = ReportErrorEventEvent;
ReportErrorEventEventHandler reportErrorEventEventHandler2;
do {
reportErrorEventEventHandler2 = reportErrorEventEventHandler;
ReportErrorEventEventHandler value2 = (ReportErrorEventEventHandler)Delegate.Combine(reportErrorEventEventHandler2, value);
reportErrorEventEventHandler = Interlocked.CompareExchange(ref ReportErrorEventEvent, value2, reportErrorEventEventHandler2);
} while ((object)reportErrorEventEventHandler != reportErrorEventEventHandler2);
}
[CompilerGenerated]
remove {
ReportErrorEventEventHandler reportErrorEventEventHandler = ReportErrorEventEvent;
ReportErrorEventEventHandler reportErrorEventEventHandler2;
do {
reportErrorEventEventHandler2 = reportErrorEventEventHandler;
ReportErrorEventEventHandler value2 = (ReportErrorEventEventHandler)Delegate.Remove(reportErrorEventEventHandler2, value);
reportErrorEventEventHandler = Interlocked.CompareExchange(ref ReportErrorEventEvent, value2, reportErrorEventEventHandler2);
} while ((object)reportErrorEventEventHandler != reportErrorEventEventHandler2);
}
}
public event DataSourcePrepEventEventHandler DataSourcePrepEvent {
[CompilerGenerated]
add {
DataSourcePrepEventEventHandler dataSourcePrepEventEventHandler = DataSourcePrepEventEvent;
DataSourcePrepEventEventHandler dataSourcePrepEventEventHandler2;
do {
dataSourcePrepEventEventHandler2 = dataSourcePrepEventEventHandler;
DataSourcePrepEventEventHandler value2 = (DataSourcePrepEventEventHandler)Delegate.Combine(dataSourcePrepEventEventHandler2, value);
dataSourcePrepEventEventHandler = Interlocked.CompareExchange(ref DataSourcePrepEventEvent, value2, dataSourcePrepEventEventHandler2);
} while ((object)dataSourcePrepEventEventHandler != dataSourcePrepEventEventHandler2);
}
[CompilerGenerated]
remove {
DataSourcePrepEventEventHandler dataSourcePrepEventEventHandler = DataSourcePrepEventEvent;
DataSourcePrepEventEventHandler dataSourcePrepEventEventHandler2;
do {
dataSourcePrepEventEventHandler2 = dataSourcePrepEventEventHandler;
DataSourcePrepEventEventHandler value2 = (DataSourcePrepEventEventHandler)Delegate.Remove(dataSourcePrepEventEventHandler2, value);
dataSourcePrepEventEventHandler = Interlocked.CompareExchange(ref DataSourcePrepEventEvent, value2, dataSourcePrepEventEventHandler2);
} while ((object)dataSourcePrepEventEventHandler != dataSourcePrepEventEventHandler2);
}
}
public event FieldMappedEventHandler FieldMapped {
[CompilerGenerated]
add {
FieldMappedEventHandler fieldMappedEventHandler = FieldMappedEvent;
FieldMappedEventHandler fieldMappedEventHandler2;
do {
fieldMappedEventHandler2 = fieldMappedEventHandler;
FieldMappedEventHandler value2 = (FieldMappedEventHandler)Delegate.Combine(fieldMappedEventHandler2, value);
fieldMappedEventHandler = Interlocked.CompareExchange(ref FieldMappedEvent, value2, fieldMappedEventHandler2);
} while ((object)fieldMappedEventHandler != fieldMappedEventHandler2);
}
[CompilerGenerated]
remove {
FieldMappedEventHandler fieldMappedEventHandler = FieldMappedEvent;
FieldMappedEventHandler fieldMappedEventHandler2;
do {
fieldMappedEventHandler2 = fieldMappedEventHandler;
FieldMappedEventHandler value2 = (FieldMappedEventHandler)Delegate.Remove(fieldMappedEventHandler2, value);
fieldMappedEventHandler = Interlocked.CompareExchange(ref FieldMappedEvent, value2, fieldMappedEventHandler2);
} while ((object)fieldMappedEventHandler != fieldMappedEventHandler2);
}
}
public BulkLoadFileImporter(LoadFile args, ProcessContext context, IIoReporter reporter, ILog logger, int timeZoneOffset, bool initializeUploaders, Guid processID, bool doRetryLogic, string bulkLoadFileFieldDelimiter, CancellationTokenSource tokenSource, Func<string> correlationIdFunc, IRunningContext runningContext = null)
: this(args, context, reporter, logger, timeZoneOffset, true, initializeUploaders, processID, doRetryLogic, bulkLoadFileFieldDelimiter, tokenSource, true, correlationIdFunc, runningContext)
{
}
public BulkLoadFileImporter(LoadFile args, ProcessContext context, IIoReporter reporter, ILog logger, int timeZoneOffset, bool autoDetect, bool initializeUploaders, Guid processID, bool doRetryLogic, string bulkLoadFileFieldDelimiter, CancellationTokenSource tokenSource, Func<string> correlationIdFunc, IRunningContext runningContext = null)
: this(args, context, reporter, logger, timeZoneOffset, autoDetect, initializeUploaders, processID, doRetryLogic, bulkLoadFileFieldDelimiter, tokenSource, true, correlationIdFunc, runningContext)
{
}
public unsafe BulkLoadFileImporter(LoadFile args, ProcessContext context, IIoReporter reporter, ILog logger, int timeZoneOffset, bool autoDetect, bool initializeUploaders, Guid processID, bool doRetryLogic, string bulkLoadFileFieldDelimiter, CancellationTokenSource tokenSource, bool initializeArtifactReader, Func<string> correlationIdFunc, IRunningContext runningContext = null)
: base(args, reporter, logger, timeZoneOffset, doRetryLogic, autoDetect, tokenSource, initializeArtifactReader, correlationIdFunc, runningContext)
{
RecordCount = -1;
Offset = 0;
DestinationFolderColumnIndex = -1;
_defaultDestinationFolderPath = string.Empty;
_runId = Guid.NewGuid().ToString().Replace("-", "_");
_lastRunMetadataImport = 0;
_batchCounter = 0;
_jobCompleteNativeCount = 0;
_jobCompleteMetadataCount = 0;
_errorLinesFileLocation = string.Empty;
MaxNumberOfErrorsInGrid = AppSettings.Instance.DefaultMaxErrorCount;
_errorCount = 0;
prePushErrorWriter = new ErrorMessageWriter<ErrorBeforeMassImportArgs>();
errorMessageFileWriter = new ErrorMessageWriter<ErrorDuringMassImportArgs>();
DisableNativeValidation = AppSettings.Instance.DisableOutsideInFileIdentification;
AuditLevel = Config.AuditLevel;
_task = null;
_usePipeliningForNativeAndObjectImports = AppSettings.Instance.UsePipeliningForNativeAndObjectImports;
_createFoldersInWebApi = AppSettings.Instance.CreateFoldersInWebApi;
_createErrorForEmptyNativeFile = AppSettings.Instance.CreateErrorForEmptyNativeFile;
base.Statistics.ImportObjectType = (TelemetryConstants.ImportObjectType)Conversions.ToInteger(Interaction.IIf(args.ArtifactTypeID == 10, TelemetryConstants.ImportObjectType.Native, TelemetryConstants.ImportObjectType.Objects));
base.ShouldImport = true;
if (string.IsNullOrEmpty(args.OverwriteDestination))
Overwrite = ImportOverwriteType.Append;
else
Overwrite = (ImportOverwriteType)Conversions.ToInteger(Enum.Parse(typeof(ImportOverwriteType), args.OverwriteDestination, true));
if (args.CopyFilesToDocumentRepository) {
string path = "EDDS" + Conversions.ToString(args.CaseInfo.ArtifactID) + "\\";
_defaultDestinationFolderPath = Path.Combine(args.SelectedCasePath, path);
if (args.ArtifactTypeID != 10) {
IEnumerator enumerator = default(IEnumerator);
try {
enumerator = args.FieldMap.GetEnumerator();
while (enumerator.MoveNext()) {
LoadFileFieldMap.LoadFileFieldMapItem loadFileFieldMapItem = (LoadFileFieldMap.LoadFileFieldMapItem)enumerator.Current;
if (loadFileFieldMapItem.DocumentField != null && loadFileFieldMapItem.NativeFileColumnIndex > -1 && loadFileFieldMapItem.DocumentField.FieldTypeID == 9) {
ref string defaultDestinationFolderPath;
*(ref defaultDestinationFolderPath = ref _defaultDestinationFolderPath) = defaultDestinationFolderPath + "File" + Conversions.ToString(loadFileFieldMapItem.DocumentField.FieldID) + "\\";
}
}
} finally {
if (enumerator is IDisposable)
(enumerator as IDisposable).Dispose();
}
}
}
if (initializeUploaders)
CreateUploaders(args);
_copyFileToRepository = args.CopyFilesToDocumentRepository;
if (!_createFoldersInWebApi && autoDetect)
_folderManager.Read(args.CaseInfo.ArtifactID, args.CaseInfo.RootFolderID);
Context = context;
FirstTimeThrough = true;
_caseInfo = args.CaseInfo;
_settings = args;
_processId = processID;
_startLineNumber = args.StartLineNumber;
OverlayArtifactId = args.IdentityFieldId;
if (string.IsNullOrEmpty(bulkLoadFileFieldDelimiter))
throw new ArgumentNullException("bulkLoadFileFieldDelimiter");
BulkLoadFileFieldDelimiter = bulkLoadFileFieldDelimiter;
_BatchSizeHistoryList = new List<int>();
}
protected virtual void CreateUploaders(LoadFile args)
{
IFileIO fileIO = ManagerFactory.CreateFileIO(args.Credentials, args.CookieContainer, GetCorrelationId);
UploadTapiBridgeParameters2 uploadTapiBridgeParameters = new UploadTapiBridgeParameters2();
uploadTapiBridgeParameters.Application = AppSettings.Instance.ApplicationName;
uploadTapiBridgeParameters.BcpFileTransfer = false;
uploadTapiBridgeParameters.AsperaBcpRootFolder = string.Empty;
uploadTapiBridgeParameters.ClientRequestId = Guid.NewGuid();
uploadTapiBridgeParameters.Credentials = (args.WebApiCredential.Credential ?? args.Credentials);
uploadTapiBridgeParameters.AsperaDocRootLevels = AppSettings.Instance.TapiAsperaNativeDocRootLevels;
uploadTapiBridgeParameters.AsperaDatagramSize = AppSettings.Instance.TapiAsperaDatagramSize;
uploadTapiBridgeParameters.FileShare = args.CaseInfo.DocumentPath;
uploadTapiBridgeParameters.ForceAsperaClient = AppSettings.Instance.TapiForceAsperaClient;
uploadTapiBridgeParameters.ForceClientCandidates = AppSettings.Instance.TapiForceClientCandidates;
uploadTapiBridgeParameters.ForceFileShareClient = AppSettings.Instance.TapiForceFileShareClient;
uploadTapiBridgeParameters.ForceHttpClient = (AppSettings.Instance.ForceWebUpload || AppSettings.Instance.TapiForceHttpClient);
uploadTapiBridgeParameters.LargeFileProgressEnabled = AppSettings.Instance.TapiLargeFileProgressEnabled;
uploadTapiBridgeParameters.LogConfigFile = AppSettings.Instance.LogConfigXmlFileName;
uploadTapiBridgeParameters.MaxFilesPerFolder = fileIO.RepositoryVolumeMax();
uploadTapiBridgeParameters.MaxInactivitySeconds = AppSettings.Instance.TapiMaxInactivitySeconds;
uploadTapiBridgeParameters.MaxJobParallelism = AppSettings.Instance.TapiMaxJobParallelism;
uploadTapiBridgeParameters.MaxJobRetryAttempts = NumberOfRetries;
uploadTapiBridgeParameters.MinDataRateMbps = AppSettings.Instance.TapiMinDataRateMbps;
uploadTapiBridgeParameters.PreserveFileTimestamps = AppSettings.Instance.TapiPreserveFileTimestamps;
uploadTapiBridgeParameters.SubmitApmMetrics = AppSettings.Instance.TapiSubmitApmMetrics;
uploadTapiBridgeParameters.TargetPath = _defaultDestinationFolderPath;
uploadTapiBridgeParameters.TargetDataRateMbps = AppSettings.Instance.TapiTargetDataRateMbps;
uploadTapiBridgeParameters.TimeoutSeconds = AppSettings.Instance.HttpTimeoutSeconds;
uploadTapiBridgeParameters.TransferLogDirectory = AppSettings.Instance.TapiTransferLogDirectory;
uploadTapiBridgeParameters.WaitTimeBetweenRetryAttempts = WaitTimeBetweenRetryAttempts;
uploadTapiBridgeParameters.WebCookieContainer = args.CookieContainer;
uploadTapiBridgeParameters.WebServiceUrl = AppSettings.Instance.WebApiServiceUrl;
uploadTapiBridgeParameters.WorkspaceId = args.CaseInfo.ArtifactID;
uploadTapiBridgeParameters.PermissionErrorsRetry = AppSettings.Instance.PermissionErrorsRetry;
uploadTapiBridgeParameters.BadPathErrorsRetry = AppSettings.Instance.TapiBadPathErrorsRetry;
uploadTapiBridgeParameters.FileNotFoundErrorsDisabled = AppSettings.Instance.TapiFileNotFoundErrorsDisabled;
uploadTapiBridgeParameters.FileNotFoundErrorsRetry = AppSettings.Instance.TapiFileNotFoundErrorsRetry;
UploadTapiBridgeParameters2 uploadTapiBridgeParameters2 = uploadTapiBridgeParameters.ShallowCopy();
uploadTapiBridgeParameters2.BcpFileTransfer = true;
uploadTapiBridgeParameters2.AsperaBcpRootFolder = AppSettings.Instance.TapiAsperaBcpRootFolder;
uploadTapiBridgeParameters2.FileShare = fileIO.GetBcpSharePath(args.CaseInfo.ArtifactID);
uploadTapiBridgeParameters2.SupportCheckPath = uploadTapiBridgeParameters2.FileShare;
uploadTapiBridgeParameters2.SortIntoVolumes = false;
uploadTapiBridgeParameters2.ForceHttpClient |= AppSettings.Instance.TapiForceBcpHttpClient;
uploadTapiBridgeParameters2.PreserveFileTimestamps = false;
CreateTapiBridges(uploadTapiBridgeParameters, uploadTapiBridgeParameters2, args.WebApiCredential.TokenProvider, new RelativityManagerServiceFactory());
}
public unsafe string[] GetColumnNames(object args)
{
string[] columnNames = _artifactReader.GetColumnNames(RuntimeHelpers.GetObjectValue(args));
checked {
int num3;
if (!_firstLineContainsColumnNames) {
int num = columnNames.Length - 1;
for (int i = 0; i <= num; i++) {
string[] array = columnNames;
int num2 = i;
num3 = i + 1;
array[num2] = "Column (" + num3.ToString() + ")";
}
} else {
num3 = columnNames.Length - 1;
for (int j = 0; j <= num3; j++) {
ref string reference;
*(ref reference = ref columnNames[j]) = reference + $"""{j + 1}""";
}
}
return columnNames;
}
}
public void DeleteTempLoadFiles()
{
if (OutputFileWriter != null)
OutputFileWriter.DeleteFiles();
}
public void WriteCodeLineToTempFile(string documentIdentifier, int codeArtifactID, int codeTypeID)
{
if (OutputFileWriter == null)
throw new InvalidOperationException($"""{codeArtifactID}""{documentIdentifier}""");
OutputFileWriter.OutputCodeFileWriter.WriteLine(string.Format("{1}{0}{2}{0}{3}{0}", BulkLoadFileFieldDelimiter, documentIdentifier, codeArtifactID, codeTypeID));
}
public void WriteObjectLineToTempFile(string ownerIdentifier, string objectName, int artifactID, int objectTypeArtifactID, int fieldID)
{
if (OutputFileWriter == null)
throw new InvalidOperationException($"""{artifactID}""{ownerIdentifier}""");
OutputFileWriter.OutputObjectFileWriter.WriteLine(string.Format("{1}{0}{2}{0}{3}{0}{4}{0}{5}{0}", BulkLoadFileFieldDelimiter, ownerIdentifier, objectName, artifactID, objectTypeArtifactID, fieldID));
}
public virtual object ReadFile(string path)
{
_filePath = path;
checked {
using (_logger.LogImportContextPushProperties(new LogContext(_runId, _settings.CaseInfo.ArtifactID)))
try {
_logger.LogUserContextInformation("Start import process", _settings.Credentials);
using (Timekeeper.CaptureTime("TOTAL")) {
OnStartFileImport();
using (Timekeeper.CaptureTime("ReadFile_InitializeMembers")) {
if (!InitializeMembers())
return false;
_processedKeyFieldValues = new Dictionary<string, int>();
}
LogInformation("Preparing to import documents via WinEDDS.");
using (Timekeeper.CaptureTime("ReadFile_ProcessDocuments")) {
_artifactReader.ValidateColumnNames(WriteInvalidColumnNameWarning);
if (_firstLineContainsColumnNames)
Offset = -1;
base.Statistics.BatchSize = ImportBatchSize;
base.JobCounter = 1;
base.TotalTransferredFilesCount = 0;
OnTapiClientChanged();
using (IFileTypeIdentifier fileTypeIdentifier = new OutsideInFileTypeIdentifierService(AppSettings.Instance.FileTypeIdentifyTimeoutSeconds)) {
while (base.ShouldImport && _artifactReader.HasMoreRecords) {
try {
if (CurrentLineNumber < _startLineNumber) {
AdvanceLine();
base.FileTapiProgressCount++;
} else {
ArtifactFieldCollection record = default(ArtifactFieldCollection);
using (Timekeeper.CaptureTime("ReadFile_GetLine")) {
WriteStatusLine(EventType2.Count, string.Empty);
record = _artifactReader.ReadArtifact();
}
long lineStatus = 0;
using (Timekeeper.CaptureTime("ReadFile_ManageDocument"))
ManageDocument(fileTypeIdentifier, record, lineStatus, CurrentLineNumber);
}
} catch (CodeCreationException ex) {
ProjectData.SetProjectError(ex);
CodeCreationException ex2 = ex;
if (ex2.IsFatal) {
WriteFatalError(CurrentLineNumber, ex2);
LogFatal(ex2, "A fatal code operation error has occurred managing an import document.");
} else {
WriteError(CurrentLineNumber, ex2.Message);
LogError(ex2, "A serious code operation error has occurred managing an import document.");
}
ProjectData.ClearProjectError();
} catch (PathTooLongException ex3) {
ProjectData.SetProjectError(ex3);
PathTooLongException exception = ex3;
WriteError(CurrentLineNumber, "Error occurred when importing the document. The folder name is longer than 255 characters.");
LogError(exception, "An import error has occured because of invalid document path - the path is too long.");
ProjectData.ClearProjectError();
} catch (ImporterException ex4) {
ProjectData.SetProjectError(ex4);
ImporterException ex5 = ex4;
WriteError(CurrentLineNumber, ex5.Message);
LogError(ex5, "An import data error has occurred managing an import document.");
ProjectData.ClearProjectError();
} catch (FileInfoInvalidPathException ex6) {
ProjectData.SetProjectError(ex6);
FileInfoInvalidPathException ex7 = ex6;
WriteError(CurrentLineNumber, ex7.Message);
LogError(ex7, "An import error has occured because of invalid document path - illegal characters in path.");
ProjectData.ClearProjectError();
} catch (FileNotFoundException ex8) {
ProjectData.SetProjectError(ex8);
FileNotFoundException ex9 = ex8;
WriteError(CurrentLineNumber, ex9.Message);
LogError(ex9, "A file not found error has occurred managing an import document.");
ProjectData.ClearProjectError();
} catch (FileTypeIdentifyException ex10) {
ProjectData.SetProjectError(ex10);
FileTypeIdentifyException ex11 = ex10;
WriteError(CurrentLineNumber, ex11.Message);
LogError(ex11, "An error occured identifying type of native file.");
ProjectData.ClearProjectError();
} catch (UnauthorizedAccessException ex12) {
ProjectData.SetProjectError(ex12);
UnauthorizedAccessException exception2 = ex12;
WriteFatalError(CurrentLineNumber, exception2);
LogFatal(exception2, "A fatal import error has occurred because the user doesn't have authorized access to the document.");
ProjectData.ClearProjectError();
} catch (Exception ex13) {
ProjectData.SetProjectError(ex13);
Exception exception3 = ex13;
WriteFatalError(CurrentLineNumber, exception3);
LogFatal(exception3, "A fatal unexpected error has occurred managing an import document.");
ProjectData.ClearProjectError();
}
}
try {
IFileTypeConfiguration configuration = fileTypeIdentifier.Configuration;
if (configuration.HasError) {
LogError("OI Configuration Info");
LogError("OI version: {Version}", configuration.Version);
LogError("OI idle worker timeout: {Timeout} seconds", configuration.Timeout);
LogError("OI install path: {InstallPath}", configuration.InstallDirectory.Secure());
if (configuration.Exception != null)
LogError(configuration.Exception, "OI runtime exception.", configuration.Exception);
} else {
LogInformation("OI Configuration Info");
LogInformation("OI version: {Version}", configuration.Version);
LogInformation("OI idle worker timeout: {Timeout}", configuration.Timeout);
LogInformation("OI install path: {InstallPath}", configuration.InstallDirectory.Secure());
}
} catch (FileTypeIdentifyException ex14) {
ProjectData.SetProjectError(ex14);
FileTypeIdentifyException exception4 = ex14;
LogError(exception4, "Failed to retrieve OI configuration info.");
ProjectData.ClearProjectError();
}
}
if (_task != null && _task.Status.In(TaskStatus.Running, TaskStatus.WaitingForActivation, TaskStatus.WaitingForChildrenToComplete, TaskStatus.WaitingToRun))
WaitOnPushBatchTask();
}
using (Timekeeper.CaptureTime("ReadFile_OtherFinalization")) {
TryPushNativeBatch(true, true, true);
WaitOnPushBatchTask();
WriteEndImport("Finish");
_artifactReader.Close();
}
}
Timekeeper.GenerateCsvReportItemsAsRows("_winedds", "C:\\");
LogInformation("Successfully imported {ImportCount} documents via WinEDDS.", base.FileTapiProgressCount);
return true;
} catch (Exception ex15) {
ProjectData.SetProjectError(ex15);
Exception exception5 = ex15;
WriteFatalError(CurrentLineNumber, exception5);
LogFatal(exception5, "A serious unexpected error has occurred importing documents.");
ProjectData.ClearProjectError();
} finally {
using (Timekeeper.CaptureTime("ReadFile_CleanupTempTables")) {
EndFileImportEvent?.Invoke(RunId);
LogStatistics();
DestroyTapiBridges();
CleanupTempTables();
if (OutputFileWriter != null) {
int num = OutputFileWriter.TryCloseAndDeleteAllTempFiles();
if (num > 0)
WriteStatusLine(EventType2.Warning, $"""{num}""", 0);
OutputFileWriter.Dispose();
OutputFileWriter = null;
}
}
}
return null;
}
}
object IImportJob.ReadFile(string path)
{
return this.ReadFile(path);
}
private bool InitializeMembers()
{
RecordCount = _artifactReader.CountRecords().GetValueOrDefault();
if (RecordCount != -1) {
InitializeFolderManagement();
InitializeFieldIdList();
DeleteTempLoadFiles();
OpenFileWriters();
OnStatusMessage(new StatusEventArgs(EventType2.ResetStartTime, 0, RecordCount, RestartTimeEventMsg, null, base.Statistics));
OnStatusMessage(new StatusEventArgs(EventType2.ResetProgress, 0, RecordCount, "Starting import...", null, base.Statistics));
return true;
}
OnStatusMessage(new StatusEventArgs(EventType2.Progress, CurrentLineNumber, CurrentLineNumber, CancelEventMsg, base.CurrentStatisticsSnapshot, base.Statistics));
return false;
}
protected override void InitializeManagers(LoadFile args)
{
base.InitializeManagers(args);
AuditManager = ManagerFactory.CreateAuditManager(args.Credentials, args.CookieContainer, GetCorrelationId);
_documentManager = ManagerFactory.CreateDocumentManager(args.Credentials, args.CookieContainer, GetCorrelationId);
RelativityManager = ManagerFactory.CreateRelativityManager(args.Credentials, args.CookieContainer, GetCorrelationId, null);
}
protected void InitializeFolderManagement()
{
checked {
if (_createFolderStructure) {
if (!_createFoldersInWebApi && _artifactTypeID == 10)
FolderCache = new FolderCache(base.Logger, _folderManager, _folderID, _caseArtifactID);
int num = _destinationFolder.LastIndexOf('(') + 1;
int num2 = _destinationFolder.LastIndexOf(')');
DestinationFolderColumnIndex = int.Parse(_destinationFolder.Substring(num, num2 - num)) - 1;
}
}
}
private void InitializeFieldIdList()
{
ArrayList arrayList = new ArrayList();
IEnumerator enumerator = default(IEnumerator);
try {
enumerator = _fieldMap.GetEnumerator();
while (enumerator.MoveNext()) {
LoadFileFieldMap.LoadFileFieldMapItem loadFileFieldMapItem = (LoadFileFieldMap.LoadFileFieldMapItem)enumerator.Current;
if (loadFileFieldMapItem.DocumentField != null && loadFileFieldMapItem.NativeFileColumnIndex != -1)
arrayList.Add(loadFileFieldMapItem.DocumentField.FieldID);
}
} finally {
if (enumerator is IDisposable)
(enumerator as IDisposable).Dispose();
}
arrayList.Add(this.get_FileInfoField(_artifactTypeID).ArtifactID);
_fieldArtifactIds = (int[])arrayList.ToArray(typeof(int));
}
private void WriteInvalidColumnNameWarning(string columnName)
{
WriteStatusLine(EventType2.Warning, $"""{columnName}""");
}
private unsafe void ManageDocument(IFileTypeIdentifier fileTypeIdentifier, ArtifactFieldCollection record, long lineStatus, int CurrentLineNumber)
{
_Closure$__136-1 closure$__136- = new _Closure$__136-1();
closure$__136-.$VB$Me = this;
closure$__136-.$VB$Local_fileTypeIdentifier = fileTypeIdentifier;
string text = string.Empty;
string text2 = string.Empty;
bool flag = record.get_FieldList(global::Relativity.DataExchange.Service.FieldType.File).Length > 0 && record.get_FieldList(global::Relativity.DataExchange.Service.FieldType.File)[0].Value != null;
closure$__136-.$VB$Local_fullFilePath = string.Empty;
IFileTypeInfo fileTypeInfo = null;
string destinationVolume = null;
IInjectableFieldCollection injectableFieldCollection = record as IInjectableFieldCollection;
bool flag2 = injectableFieldCollection == null;
checked {
bool flag3 = default(bool);
using (Timekeeper.CaptureTime("ManageDocument_Filesystem")) {
if (flag && _artifactTypeID == 10) {
text = record.get_FieldList(global::Relativity.DataExchange.Service.FieldType.File)[0].Value.ToString();
if (text.Length > 1 && Operators.CompareString(Conversions.ToString(text[0]), "\\", false) == 0 && Operators.CompareString(Conversions.ToString(text[1]), "\\", false) != 0)
text = "." + text;
string path = text;
if (base.DisableNativeLocationValidation)
flag3 = true;
else {
string existingFilePath = GetExistingFilePath(text, true);
flag3 = !string.IsNullOrEmpty(existingFilePath);
if (flag3 && !string.Equals(text, existingFilePath)) {
WriteWarning($"""{text}""{existingFilePath}""");
text = existingFilePath;
}
}
if (text.Trim().Equals(string.Empty))
flag3 = false;
if (Operators.CompareString(text, string.Empty, false) != 0 && !flag3)
lineStatus += 256;
if (flag3 && !base.DisableNativeLocationValidation && GetFileLength(text, true) == 0) {
if (_createErrorForEmptyNativeFile)
lineStatus += 2048;
else
WriteWarning($"""{text}""");
}
closure$__136-.$VB$Local_fullFilePath = text;
if (flag3) {
DateTime now = DateTime.Now;
try {
if (DisableNativeValidation)
fileTypeInfo = null;
else {
IHasOixFileType hasOixFileType = null;
if (!flag2)
hasOixFileType = injectableFieldCollection.FileIdInfo;
if (hasOixFileType == null) {
_Closure$__136-0 closure$__136-2;
closure$__136-2.$VB$NonLocal_$VB$Closure_2 = closure$__136-;
closure$__136-2.$VB$Local_maxRetryAttempts = NumberOfRetries;
closure$__136-2.$VB$Local_currentRetryAttempt = 0;
fileTypeInfo = CreateWaitAndRetryPolicy().WaitAndRetry(delegate(Exception exception) {
FileTypeIdentifyException ex3 = exception as FileTypeIdentifyException;
if (ex3 == null)
return RetryExceptionHelper.IsRetryable(exception, base.RetryOptions);
if (ex3.Error != FileTypeIdentifyError.Permissions)
return true;
return base.RetryOptions.HasFlag(RetryOptions.Permissions);
}, delegate(int count) {
closure$__136-2.$VB$Local_currentRetryAttempt = count;
if (count > 1)
closure$__136-2.$VB$NonLocal_$VB$Closure_2.$VB$Local_fileTypeIdentifier.Reinitialize();
return TimeSpan.FromSeconds(unchecked((double)closure$__136-2.$VB$NonLocal_$VB$Closure_2.$VB$Me.WaitTimeBetweenRetryAttempts));
}, delegate(Exception exception, TimeSpan span) {
closure$__136-2.$VB$NonLocal_$VB$Closure_2.$VB$Me.PublishIoRetryMessage(exception, span, closure$__136-2.$VB$Local_currentRetryAttempt, closure$__136-2.$VB$Local_maxRetryAttempts);
}, (CancellationToken a0) => closure$__136-2.$VB$NonLocal_$VB$Closure_2._Lambda$__3(), base.CancellationToken);
} else
fileTypeInfo = hasOixFileType.GetFileTypeIdInfo();
}
Guid guid;
if (_copyFileToRepository) {
if ((!base.DisableNativeLocationValidation && flag3) || GetFileExists(text)) {
guid = Guid.NewGuid();
string targetFileName = guid.ToString();
base.ImportFilesCount++;
ref int jobCompleteNativeCount;
*(ref jobCompleteNativeCount = ref _jobCompleteNativeCount) = jobCompleteNativeCount + 1;
text2 = base.FileTapiBridge.AddPath(text, targetFileName, this.CurrentLineNumber);
destinationVolume = base.FileTapiBridge.TargetFolderName;
} else
WriteWarning($"""{text}""");
} else {
guid = Guid.NewGuid();
text2 = guid.ToString();
}
text = ((flag2 || !injectableFieldCollection.HasFileName()) ? Path.GetFileName(path) : injectableFieldCollection.FileName.GetFileName());
} catch (FileNotFoundException ex) {
ProjectData.SetProjectError(ex);
FileNotFoundException ex2 = ex;
if (!base.DisableNativeLocationValidation)
throw;
ProjectData.ClearProjectError();
}
if (base.ShouldImport && !_copyFileToRepository)
WriteStatusLine(EventType2.Status, $"""{(DateTime.Now - now).Milliseconds}""");
}
}
}
string folderPath = string.Empty;
int parentFolderID = default(int);
using (Timekeeper.CaptureTime("ManageDocument_Folder")) {
if (!_createFolderStructure)
parentFolderID = ((_artifactTypeID != 10 && ParentArtifactTypeID != 8) ? (-1) : _folderID);
else if (_artifactTypeID == 10) {
string path2 = NullableTypesHelper.ToEmptyStringOrValue(NullableTypesHelper.DBNullString(RuntimeHelpers.GetObjectValue(record.get_FieldList(global::Relativity.DataExchange.Service.FieldCategory.ParentArtifact)[0].Value)));
if (_createFoldersInWebApi) {
string text3 = CleanDestinationFolderPath(path2);
if (string.IsNullOrWhiteSpace(text3))
parentFolderID = _folderID;
else {
if (InnerRelativityFolderPathsAreTooLarge(text3))
throw new PathTooLongException("Error occurred when importing the document. The folder name is longer than 255 characters.");
folderPath = text3;
parentFolderID = -9;
}
} else
parentFolderID = FolderCache.GetFolderId(CleanDestinationFolderPath(path2));
} else {
string text4 = NullableTypesHelper.ToEmptyStringOrValue(NullableTypesHelper.DBNullString(record.get_FieldList(global::Relativity.DataExchange.Service.FieldCategory.ParentArtifact)[0].Value.ToString()));
if (Operators.CompareString(text4, "", false) == 0)
throw new ParentObjectReferenceRequiredException(this.CurrentLineNumber, DestinationFolderColumnIndex);
DataTable dataTable = _objectManager.RetrieveArtifactIdOfMappedParentObject(_caseArtifactID, text4, _artifactTypeID).Tables[0];
if (dataTable.Rows.Count > 1)
throw new DuplicateObjectReferenceException(this.CurrentLineNumber, DestinationFolderColumnIndex, "Parent Info");
if (dataTable.Rows.Count == 0)
throw new NonExistentParentException(this.CurrentLineNumber, DestinationFolderColumnIndex, "Parent Info");
parentFolderID = Conversions.ToInteger(dataTable.Rows[0]["ArtifactID"]);
}
}
string identityValue = PrepareFieldCollectionAndExtractIdentityValue(record, CurrentLineNumber);
string dataGridID = null;
ArtifactField artifactField = record.get_FieldList(global::Relativity.DataExchange.Service.FieldType.Varchar).FirstOrDefault((_Closure$__.$I136-4 != null) ? _Closure$__.$I136-4 : (_Closure$__.$I136-4 = ((ArtifactField x) => Operators.CompareString(x.DisplayName, "DataGridID", false) == 0)));
if (artifactField != null)
dataGridID = artifactField.ValueAsString;
IHasFileSize hasFileSize = null;
if (!flag2)
hasFileSize = injectableFieldCollection.FileSize;
MetaDocument metaDoc = (hasFileSize != null) ? new SizedMetaDocument(text2, identityValue, flag3 && flag && (Operators.CompareString(text2, string.Empty, false) != 0 || !_copyFileToRepository), text, closure$__136-.$VB$Local_fullFilePath, flag, CurrentLineNumber, parentFolderID, record, fileTypeInfo, lineStatus, destinationVolume, hasFileSize.GetFileSize(), folderPath, dataGridID) : new MetaDocument(text2, identityValue, flag3 && flag && (Operators.CompareString(text2, string.Empty, false) != 0 || !_copyFileToRepository), text, closure$__136-.$VB$Local_fullFilePath, flag, CurrentLineNumber, parentFolderID, record, fileTypeInfo, lineStatus, destinationVolume, folderPath, dataGridID);
using (Timekeeper.CaptureTime("ManageDocument_ManageDocumentMetadata"))
ManageDocumentMetaData(metaDoc);
}
}
public static string CleanDestinationFolderPath(string path)
{
path = path.Trim();
while (path.Contains(".\\")) {
path = path.Replace(".\\", "\\");
}
while (path.Contains("\\\\")) {
path = path.Replace("\\\\", "\\");
}
path = path.Replace(":", "_");
if (path.Length != 0 && path[0] != '\\')
path = "\\" + path;
path = path.TrimEnd(new char[1] {
'\\'
});
if (string.IsNullOrWhiteSpace(path))
path = "\\";
return path;
}
public static bool InnerRelativityFolderPathsAreTooLarge(string cleanFolderPath)
{
if (!string.IsNullOrEmpty(cleanFolderPath)) {
if (!cleanFolderPath.Contains("\\") && cleanFolderPath.Length <= 255)
return false;
return cleanFolderPath.Split(new char[1] {
"\\"[0]
}).Any((_Closure$__.$I138-0 != null) ? _Closure$__.$I138-0 : (_Closure$__.$I138-0 = ((string path) => path.Length > 255)));
}
return false;
}
public override int LookupArtifactIDForName(string objectName, int associatedObjectTypeID)
{
if (associatedObjectTypeID != 10)
return -1;
return base.LookupArtifactIDForName(objectName, associatedObjectTypeID);
}
public override string LookupNameForArtifactID(int objectArtifactID, int associatedObjectTypeID)
{
if (associatedObjectTypeID != 10)
return string.Empty;
return base.LookupNameForArtifactID(objectArtifactID, associatedObjectTypeID);
}
private unsafe void ManageDocumentMetaData(MetaDocument metaDoc)
{
checked {
try {
using (Timekeeper.CaptureTime("ManageDocumentMetadata_ManageDocumentLine"))
ManageDocumentLine(metaDoc);
ref int batchCounter;
*(ref batchCounter = ref _batchCounter) = batchCounter + 1;
using (Timekeeper.CaptureTime("ManageDocumentMetadata_WserviceCall")) {
if (OutputFileWriter.CombinedStreamLength > ImportBatchVolume || _batchCounter > ImportBatchSize - 1)
TryPushNativeBatch(false, _jobCompleteNativeCount >= JobCompleteBatchSize, _jobCompleteMetadataCount >= JobCompleteBatchSize);
}
} catch (ImporterException ex) {
ProjectData.SetProjectError(ex);
ImporterException ex2 = ex;
WriteError(metaDoc.LineNumber, ex2.Message);
LogError(ex2, "A serious import error has occurred managing document {file} metadata.", metaDoc.FullFilePath.Secure());
ProjectData.ClearProjectError();
} catch (FileInfoInvalidPathException ex3) {
ProjectData.SetProjectError(ex3);
FileInfoInvalidPathException ex4 = ex3;
WriteError(metaDoc.LineNumber, ex4.Message);
LogError(ex4, "An import error has occured because of invalid document path - illegal characters in path {0}", metaDoc.FullFilePath.Secure());
ProjectData.ClearProjectError();
} catch (UnauthorizedAccessException ex5) {
ProjectData.SetProjectError(ex5);
UnauthorizedAccessException exception = ex5;
WriteFatalError(metaDoc.LineNumber, exception);
LogFatal(exception, "A fatal import error has occurred because the user doesn't have authorized access to the document {file} metadata.", metaDoc.FullFilePath.Secure());
ProjectData.ClearProjectError();
} catch (Exception ex6) {
ProjectData.SetProjectError(ex6);
Exception exception2 = ex6;
WriteFatalError(metaDoc.LineNumber, exception2);
LogFatal(exception2, "A fatal unexpected error has occurred managing document {file} metadata.", metaDoc.FullFilePath.Secure());
ProjectData.ClearProjectError();
}
if (_copyFileToRepository && metaDoc.IndexFileInDB) {
using (Timekeeper.CaptureTime("ManageDocumentMetadata_StatusEvent"))
WriteStatusLine(EventType2.Status, $"""{metaDoc.IdentityValue}""{metaDoc.FileGuid}""", metaDoc.LineNumber);
} else {
using (Timekeeper.CaptureTime("ManageDocumentMetadata_ProgressEvent")) {
base.FileTapiProgressCount++;
WriteStatusLine(EventType2.Progress, $"""{metaDoc.IdentityValue}""{metaDoc.FileGuid}""", metaDoc.LineNumber);
}
}
}
}
protected MassImportResults BulkImport(NativeLoadInfo settings, bool includeExtractedTextEncoding)
{
if (BatchSizeHistoryList.Count == 0)
BatchSizeHistoryList.Add(ImportBatchSize);
try {
return BatchBulkImport(settings, includeExtractedTextEncoding);
} catch (Exception ex) {
ProjectData.SetProjectError(ex);
Exception ex2 = ex;
if (base.ShouldImport) {
string bulkImportErrorMessage = GetBulkImportErrorMessage(ex2);
LogFatal(ex2, bulkImportErrorMessage);
}
throw;
}
}
private string GetBulkImportErrorMessage(Exception ex)
{
if (!ImportTapiBase.IsTimeoutException(ex)) {
if (!ImportTapiBase.IsBulkImportSqlException(ex)) {
if (!ImportTapiBase.IsInsufficientPermissionsForImportException(ex))
return "A serious error has occurred bulk importing the native batch.";
return "A fatal insufficient permissions error has occurred bulk importing the native batch.";
}
return "A fatal SQL error has occurred bulk importing the native batch.";
}
return "A SQL or HTTP timeout error has occurred bulk importing the native batch.";
}
private MassImportResults BatchBulkImport(NativeLoadInfo settings, bool includeExtractedTextEncoding)
{
if (!(settings is ObjectLoadInfo))
return BulkImportManager.BulkImportNative(_caseInfo.ArtifactID, settings, _copyFileToRepository, includeExtractedTextEncoding);
return BulkImportManager.BulkImportObjects(_caseInfo.ArtifactID, (ObjectLoadInfo)settings, _copyFileToRepository);
}
protected virtual void LowerBatchLimits()
{
int importBatchSize = ImportBatchSize;
checked {
ImportBatchSize -= 100;
base.Statistics.BatchSize = ImportBatchSize;
BatchSizeHistoryList.Add(ImportBatchSize);
LogWarning("Lowered the native batch limits from {OldBatchSize} to {NewBatchSize}.", importBatchSize, ImportBatchSize);
}
}
private NativeLoadInfo GetSettingsObject()
{
NativeLoadInfo nativeLoadInfo;
if (_artifactTypeID == 10) {
nativeLoadInfo = new NativeLoadInfo {
DisableUserSecurityCheck = DisableUserSecurityCheck,
AuditLevel = AuditLevel,
OverlayArtifactID = OverlayArtifactId
};
if (_createFoldersInWebApi) {
nativeLoadInfo.RootFolderID = _folderID;
if (nativeLoadInfo.RootFolderID == 0)
nativeLoadInfo.RootFolderID = -1;
} else
nativeLoadInfo.RootFolderID = 0;
} else
nativeLoadInfo = new ObjectLoadInfo {
DisableUserSecurityCheck = DisableUserSecurityCheck,
AuditLevel = AuditLevel,
ArtifactTypeID = _artifactTypeID
};
nativeLoadInfo.LinkDataGridRecords = LinkDataGridRecords;
OnSettingsObjectCreate(nativeLoadInfo);
return nativeLoadInfo;
}
public virtual void OnSettingsObjectCreate(NativeLoadInfo settings)
{
}
private void TryPushNativeBatch(bool lastRun, bool shouldCompleteNativeJob, bool shouldCompleteMetadataJob)
{
CloseFileWriters();
string outputNativeFilePath = OutputFileWriter.OutputNativeFilePath;
if ((shouldCompleteNativeJob | lastRun) & (_jobCompleteNativeCount > 0)) {
_jobCompleteNativeCount = 0;
AwaitPendingPhysicalFileUploadsForJob();
}
checked {
if (base.ShouldImport)
try {
if (base.ShouldImport && _copyFileToRepository && base.FileTapiBridge.TransfersPending) {
AwaitPendingPhysicalFileUploadsForBatch();
base.JobCounter++;
int num = CurrentLineNumber + Offset;
if (base.ShouldImport && base.FileTapiProgressCount != num) {
base.FileTapiProgressCount = num;
WriteTapiProgressMessage("Synchronized process count.", CurrentLineNumber);
}
}
DateTime now = DateTime.Now;
long ticks = now.Ticks;
if (base.ShouldImport)
PushNativeBatch(outputNativeFilePath, shouldCompleteMetadataJob, lastRun);
ImportStatistics statistics;
ImportStatistics importStatistics = statistics = base.Statistics;
TimeSpan fileWaitDuration = statistics.FileWaitDuration;
now = DateTime.Now;
importStatistics.FileWaitDuration = fileWaitDuration + new TimeSpan(Math.Max(now.Ticks - ticks, 1));
} catch (Exception ex) {
ProjectData.SetProjectError(ex);
Exception ex2 = ex;
if (base.ShouldImport)
LogError("Pushing the native batch failed but lowering the batch and performing a retry is disabled.", ex2);
throw;
}
DeleteTempLoadFiles();
if (!lastRun)
OpenFileWriters();
}
}
private void LowerBatchSizeAndRetry(string oldNativeFilePath, int totalRecords)
{
string tempFileName = TempFileBuilder.GetTempFileName("rel-native");
string text = BulkLoadFileFieldDelimiter + "\r\n";
Queue<char> queue = new Queue<char>();
int num = 0;
long num2 = 0;
bool flag = false;
int num3 = 1;
checked {
while (totalRecords > num && !flag && base.ShouldImport) {
int num4 = 0;
long num5 = 0;
using (StreamReader streamReader = new StreamReader(oldNativeFilePath, Encoding.Unicode))
using (StreamWriter streamWriter = new StreamWriter(tempFileName, false, Encoding.Unicode)) {
AdvanceStream(streamReader, num2);
while (num4 < ImportBatchSize && !flag) {
char c = Strings.ChrW(streamReader.Read());
queue.Enqueue(c);
if (queue.Count > text.Length)
queue.Dequeue();
if (Operators.CompareString(new string(queue.ToArray()), text, false) == 0) {
streamWriter.Flush();
num4++;
}
streamWriter.Write(c);
num5++;
flag = (streamReader.Peek() == -1);
}
streamWriter.Flush();
}
try {
_batchCounter = num4;
WriteWarning("Processing sub-batch of size " + Conversions.ToString(ImportBatchSize) + ". " + Conversions.ToString(num) + " of " + Conversions.ToString(totalRecords) + " in the original batch processed");
PushNativeBatch(tempFileName, false, true);
num += num4;
num2 += num5;
} catch (Exception ex) {
ProjectData.SetProjectError(ex);
Exception ex2 = ex;
if (num3 >= NumberOfRetries || !BatchResizeEnabled || !ImportTapiBase.IsTimeoutException(ex2) || !base.ShouldImport) {
global::Relativity.DataExchange.Io.FileSystem.Instance.File.Delete(tempFileName);
throw;
}
LogWarning(ex2, "A serious SQL or HTTP timeout error has occurred and the native batch will be resized.");
LowerBatchLimits();
RaiseWarningAndPause(ex2, WaitTimeBetweenRetryAttempts, num3, NumberOfRetries);
if (!base.ShouldImport)
throw;
num3++;
flag = false;
ProjectData.ClearProjectError();
}
}
global::Relativity.DataExchange.Io.FileSystem.Instance.File.Delete(tempFileName);
}
}
private void AdvanceStream(StreamReader sr, long count)
{
checked {
if (count > 0) {
long num = count - 1;
for (long num2 = 0; num2 <= num; num2++) {
sr.Read();
}
}
}
}
protected unsafe void PushNativeBatch(string outputNativePath, bool shouldCompleteJob, bool lastRun)
{
checked {
DateTime now;
if (_lastRunMetadataImport > 0) {
ImportStatistics statistics;
ImportStatistics importStatistics = statistics = base.Statistics;
TimeSpan metadataWaitDuration = statistics.MetadataWaitDuration;
now = DateTime.Now;
importStatistics.MetadataWaitDuration = metadataWaitDuration + new TimeSpan(now.Ticks - _lastRunMetadataImport);
}
if (_batchCounter == 0 || !base.ShouldImport) {
if (_jobCompleteMetadataCount > 0) {
_jobCompleteMetadataCount = 0;
AwaitPendingBulkLoadFileUploadsForJob();
}
} else {
_batchCounter = 0;
if (shouldCompleteJob & (_jobCompleteMetadataCount > 0)) {
_jobCompleteMetadataCount = 0;
AwaitPendingBulkLoadFileUploadsForJob();
}
_Closure$__151-0 closure$__151-;
closure$__151-.$VB$Local_settings = GetSettingsObject();
closure$__151-.$VB$Local_settings.UseBulkDataImport = true;
UploadTapiBridge2 bulkLoadTapiBridge = base.BulkLoadTapiBridge;
Guid guid = Guid.NewGuid();
string dataFileName = bulkLoadTapiBridge.AddPath(outputNativePath, guid.ToString(), 1);
UploadTapiBridge2 bulkLoadTapiBridge2 = base.BulkLoadTapiBridge;
string outputCodeFilePath = OutputFileWriter.OutputCodeFilePath;
guid = Guid.NewGuid();
string codeFileName = bulkLoadTapiBridge2.AddPath(outputCodeFilePath, guid.ToString(), 2);
UploadTapiBridge2 bulkLoadTapiBridge3 = base.BulkLoadTapiBridge;
string outputObjectFilePath = OutputFileWriter.OutputObjectFilePath;
guid = Guid.NewGuid();
string objectFileName = bulkLoadTapiBridge3.AddPath(outputObjectFilePath, guid.ToString(), 3);
UploadTapiBridge2 bulkLoadTapiBridge4 = base.BulkLoadTapiBridge;
string outputDataGridFilePath = OutputFileWriter.OutputDataGridFilePath;
guid = Guid.NewGuid();
string dataGridFileName = bulkLoadTapiBridge4.AddPath(outputDataGridFilePath, guid.ToString(), 4);
base.MetadataFilesCount += 4;
ref int jobCompleteMetadataCount;
*(ref jobCompleteMetadataCount = ref _jobCompleteMetadataCount) = jobCompleteMetadataCount + 4;
if (lastRun)
AwaitPendingBulkLoadFileUploadsForJob();
else
AwaitPendingBulkLoadFileUploadsForBatch();
now = DateTime.Now;
_lastRunMetadataImport = now.Ticks;
if (base.ShouldImport) {
if (_artifactTypeID == 10) {
closure$__151-.$VB$Local_settings.Repository = _defaultDestinationFolderPath;
if (Operators.CompareString(closure$__151-.$VB$Local_settings.Repository, string.Empty, false) == 0)
closure$__151-.$VB$Local_settings.Repository = _caseInfo.DocumentPath;
} else
closure$__151-.$VB$Local_settings.Repository = _caseInfo.DocumentPath;
closure$__151-.$VB$Local_settings.RunID = _runId;
closure$__151-.$VB$Local_settings.CodeFileName = codeFileName;
closure$__151-.$VB$Local_settings.DataFileName = dataFileName;
closure$__151-.$VB$Local_settings.ObjectFileName = objectFileName;
closure$__151-.$VB$Local_settings.DataGridFileName = dataGridFileName;
closure$__151-.$VB$Local_settings.MappedFields = GetMappedFields(_artifactTypeID, _settings.ObjectFieldIdListContainsArtifactId);
closure$__151-.$VB$Local_settings.KeyFieldArtifactID = _keyFieldID;
closure$__151-.$VB$Local_settings.BulkLoadFileFieldDelimiter = BulkLoadFileFieldDelimiter;
closure$__151-.$VB$Local_settings.OverlayBehavior = GetMassImportOverlayBehavior(_settings.OverlayBehavior);
closure$__151-.$VB$Local_settings.MoveDocumentsInAppendOverlayMode = _settings.MoveDocumentsInAppendOverlayMode;
switch (Overwrite) {
case ImportOverwriteType.Overlay:
closure$__151-.$VB$Local_settings.Overlay = OverwriteType.Overlay;
break;
case ImportOverwriteType.AppendOverlay:
closure$__151-.$VB$Local_settings.Overlay = OverwriteType.Both;
break;
default:
closure$__151-.$VB$Local_settings.Overlay = OverwriteType.Append;
break;
}
closure$__151-.$VB$Local_settings.UploadFiles = (_filePathColumnIndex != -1 && _settings.LoadNativeFiles);
closure$__151-.$VB$Local_settings.LoadImportedFullTextFromServer = base.LoadImportedFullTextFromServer;
closure$__151-.$VB$Local_settings.ExecutionSource = unchecked((kCura.EDDS.WebAPI.BulkImportManagerBase.ExecutionSource)_executionSource);
closure$__151-.$VB$Local_settings.Billable = _settings.Billable;
if (_usePipeliningForNativeAndObjectImports && _task != null) {
WaitOnPushBatchTask();
_task = null;
}
Action action = delegate {
DateTime now2 = DateTime.Now;
MassImportResults massImportResults = BulkImport(closure$__151-.$VB$Local_settings, _fullTextColumnMapsToFileLocation);
base.Statistics.ProcessMassImportResults(massImportResults);
TimeSpan timeSpan = DateTime.Now - now2;
ImportStatistics statistics2;
(statistics2 = base.Statistics).MassImportDuration = statistics2.MassImportDuration + timeSpan;
(statistics2 = base.Statistics).BatchCount = statistics2.BatchCount + 1;
base.Logger.LogInformation("Duration of mass import processing: {durationInMilliseconds}, batch: {numberOfBatch}", new object[2] {
timeSpan.TotalMilliseconds,
base.Statistics.BatchCount
});
UpdateStatisticsSnapshot(DateTime.Now, false);
ManageErrors(_artifactTypeID);
BatchInformation batchInformation = new BatchInformation {
OrdinalNumber = base.Statistics.BatchCount,
NumberOfRecords = massImportResults.FilesProcessed,
MassImportDuration = timeSpan
};
base.OnBatchCompleted(batchInformation);
};
if (_usePipeliningForNativeAndObjectImports) {
TaskFactory taskFactory = new TaskFactory();
_task = taskFactory.StartNew(action);
} else
action();
if (_firstLineContainsColumnNames && _startLineNumber > 1)
base.TotalTransferredFilesCount = base.FileTapiProgressCount - (int)_startLineNumber + 1;
else if (!_firstLineContainsColumnNames && _startLineNumber > 0) {
base.TotalTransferredFilesCount = base.FileTapiProgressCount - (int)_startLineNumber;
} else {
base.TotalTransferredFilesCount = base.FileTapiProgressCount;
}
}
}
}
}
private void WaitOnPushBatchTask()
{
if (_task != null)
try {
Task.WaitAll(_task);
} catch (AggregateException ex) {
ProjectData.SetProjectError(ex);
AggregateException ex2 = ex;
LogFatal(ex2, "A fatal error occurred while waiting on the batch task");
ex2.Handle((_Closure$__.$I152-0 != null) ? _Closure$__.$I152-0 : (_Closure$__.$I152-0 = delegate(Exception e) {
throw e;
}));
ProjectData.ClearProjectError();
}
}
protected OverlayBehavior GetMassImportOverlayBehavior(LoadFile.FieldOverlayBehavior? inputOverlayType)
{
LoadFile.FieldOverlayBehavior? nullable = inputOverlayType;
int? nullable2 = (int?)nullable;
if (!(nullable2.HasValue ? new bool?(nullable2.GetValueOrDefault() == 1) : null).GetValueOrDefault()) {
nullable2 = (int?)nullable;
if (!(nullable2.HasValue ? new bool?(nullable2.GetValueOrDefault() == 2) : null).GetValueOrDefault())
return OverlayBehavior.UseRelativityDefaults;
return OverlayBehavior.ReplaceAll;
}
return OverlayBehavior.MergeAll;
}
protected void OpenFileWriters()
{
if (OutputFileWriter == null)
OutputFileWriter = new OutputFileWriter(base.Logger, base.FileSystem);
OutputFileWriter.Open(false);
}
protected void CloseFileWriters()
{
if (OutputFileWriter != null)
OutputFileWriter.Close();
}
public FieldInfo[] GetMappedFields(int artifactTypeId, IList<int> ObjectFieldIdListContainsArtifactId)
{
ArrayList arrayList = new ArrayList();
IEnumerator enumerator = default(IEnumerator);
try {
enumerator = _fieldMap.GetEnumerator();
while (enumerator.MoveNext()) {
LoadFileFieldMap.LoadFileFieldMapItem loadFileFieldMapItem = (LoadFileFieldMap.LoadFileFieldMapItem)enumerator.Current;
if (loadFileFieldMapItem.DocumentField != null) {
int index = arrayList.Add(loadFileFieldMapItem.DocumentField.ToFieldInfo());
if (ObjectFieldIdListContainsArtifactId != null && ((((FieldInfo)arrayList[index]).Type == kCura.EDDS.WebAPI.BulkImportManagerBase.FieldType.Object) | (((FieldInfo)arrayList[index]).Type == kCura.EDDS.WebAPI.BulkImportManagerBase.FieldType.Objects)) && ObjectFieldIdListContainsArtifactId.Contains(((FieldInfo)arrayList[index]).ArtifactID))
((FieldInfo)arrayList[index]).ImportBehavior = kCura.EDDS.WebAPI.BulkImportManagerBase.ImportBehaviorChoice.ObjectFieldContainsArtifactId;
}
}
} finally {
if (enumerator is IDisposable)
(enumerator as IDisposable).Dispose();
}
arrayList.Sort(new WebServiceFieldInfoNameComparer());
if (artifactTypeId == 10) {
arrayList.Add(GetIsSupportedRelativityFileTypeField());
arrayList.Add(GetRelativityFileTypeField());
arrayList.Add(GetHasNativesField());
}
return (FieldInfo[])arrayList.ToArray(typeof(FieldInfo));
}
protected void ManageDocumentLine(MetaDocument mdoc)
{
Encoding chosenEncoding = null;
OutputFileWriter.MarkRollbackPosition();
OutputFileWriter.OutputNativeFileWriter.Write("0" + BulkLoadFileFieldDelimiter);
OutputFileWriter.OutputNativeFileWriter.Write(mdoc.LineStatus.ToString() + BulkLoadFileFieldDelimiter);
OutputFileWriter.OutputNativeFileWriter.Write("0" + BulkLoadFileFieldDelimiter);
OutputFileWriter.OutputNativeFileWriter.Write("0" + BulkLoadFileFieldDelimiter);
OutputFileWriter.OutputNativeFileWriter.Write(Conversions.ToString(mdoc.LineNumber) + BulkLoadFileFieldDelimiter);
if (mdoc.UploadFile & mdoc.IndexFileInDB) {
OutputFileWriter.OutputNativeFileWriter.Write(mdoc.FileGuid + BulkLoadFileFieldDelimiter);
OutputFileWriter.OutputNativeFileWriter.Write(mdoc.Filename + BulkLoadFileFieldDelimiter);
if (_settings.CopyFilesToDocumentRepository) {
OutputFileWriter.OutputNativeFileWriter.Write(_defaultDestinationFolderPath + mdoc.DestinationVolume + "\\" + mdoc.FileGuid + BulkLoadFileFieldDelimiter);
OutputFileWriter.OutputNativeFileWriter.Write(mdoc.FullFilePath + BulkLoadFileFieldDelimiter);
} else {
OutputFileWriter.OutputNativeFileWriter.Write(mdoc.FullFilePath + BulkLoadFileFieldDelimiter);
OutputFileWriter.OutputNativeFileWriter.Write(mdoc.FullFilePath + BulkLoadFileFieldDelimiter);
}
IHasFileSize hasFileSize = mdoc as IHasFileSize;
if (hasFileSize == null) {
if (GetFileExists(mdoc.FullFilePath))
OutputFileWriter.OutputNativeFileWriter.Write(Conversions.ToString(GetFileLength(mdoc.FullFilePath, true)) + BulkLoadFileFieldDelimiter);
else
OutputFileWriter.OutputNativeFileWriter.Write(Conversions.ToString(0) + BulkLoadFileFieldDelimiter);
} else
OutputFileWriter.OutputNativeFileWriter.Write(Conversions.ToString(hasFileSize.GetFileSize()) + BulkLoadFileFieldDelimiter);
} else {
OutputFileWriter.OutputNativeFileWriter.Write(BulkLoadFileFieldDelimiter);
OutputFileWriter.OutputNativeFileWriter.Write(BulkLoadFileFieldDelimiter);
OutputFileWriter.OutputNativeFileWriter.Write(BulkLoadFileFieldDelimiter);
OutputFileWriter.OutputNativeFileWriter.Write(BulkLoadFileFieldDelimiter);
OutputFileWriter.OutputNativeFileWriter.Write("0" + BulkLoadFileFieldDelimiter);
}
OutputFileWriter.OutputNativeFileWriter.Write(Conversions.ToString(mdoc.ParentFolderID) + BulkLoadFileFieldDelimiter);
bool flag = false;
if (!string.IsNullOrEmpty(mdoc.DataGridID)) {
OutputFileWriter.OutputDataGridFileWriter.Write(mdoc.IdentityValue + BulkLoadFileFieldDelimiter + mdoc.DataGridID + BulkLoadFileFieldDelimiter);
flag = true;
}
foreach (ArtifactField item in mdoc.Record) {
try {
switch (item.EnableDataGrid) {
case false:
WriteDocumentField(ref chosenEncoding, item, OutputFileWriter.OutputNativeFileWriter, BulkLoadFileFieldDelimiter, _artifactTypeID, _extractedTextFileEncoding);
break;
case true:
if (!flag) {
OutputFileWriter.OutputDataGridFileWriter.Write(mdoc.IdentityValue + BulkLoadFileFieldDelimiter + string.Empty + BulkLoadFileFieldDelimiter);
flag = true;
}
WriteDocumentField(ref chosenEncoding, item, OutputFileWriter.OutputDataGridFileWriter, BulkLoadFileFieldDelimiter, _artifactTypeID, _extractedTextFileEncoding);
break;
}
} catch (ExtractedTextFileNotFoundException ex) {
ProjectData.SetProjectError(ex);
ExtractedTextFileNotFoundException ex2 = ex;
OutputFileWriter.RollbackDocumentLineWrites();
throw;
} catch (ExtractedTextTooLargeException ex3) {
ProjectData.SetProjectError(ex3);
ExtractedTextTooLargeException ex4 = ex3;
OutputFileWriter.RollbackDocumentLineWrites();
throw;
} catch (ImportIOException ex5) {
ProjectData.SetProjectError(ex5);
ImportIOException ex6 = ex5;
OutputFileWriter.RollbackDocumentLineWrites();
throw;
}
}
if (_artifactTypeID == 10)
WriteDocumentNativeInfo(mdoc);
if (chosenEncoding != null)
OutputFileWriter.OutputNativeFileWriter.Write(chosenEncoding.CodePage.ToString() + BulkLoadFileFieldDelimiter);
else if (_fullTextColumnMapsToFileLocation) {
OutputFileWriter.OutputNativeFileWriter.Write(BulkLoadFileFieldDelimiter);
}
if (_createFoldersInWebApi && _artifactTypeID == 10)
OutputFileWriter.OutputNativeFileWriter.Write(mdoc.FolderPath + BulkLoadFileFieldDelimiter);
OutputFileWriter.OutputNativeFileWriter.Write(BulkLoadFileFieldDelimiter);
OutputFileWriter.OutputNativeFileWriter.Write(BulkLoadFileFieldDelimiter);
OutputFileWriter.OutputNativeFileWriter.Write("\r\n");
if (flag)
OutputFileWriter.OutputDataGridFileWriter.Write("\r\n");
}
private void WriteDocumentNativeInfo(MetaDocument mdoc)
{
if (_filePathColumnIndex != -1 && mdoc.UploadFile && mdoc.IndexFileInDB) {
IHasSupportedByViewer hasSupportedByViewer = mdoc.FileTypeInfo as IHasSupportedByViewer;
if (hasSupportedByViewer == null)
WriteDocumentNativeInfo(IsSupportedRelativityFileType(mdoc.FileTypeInfo), mdoc.GetFileType(), true);
else
WriteDocumentNativeInfo(hasSupportedByViewer.SupportedByViewer(), mdoc.GetFileType(), true);
} else
WriteDocumentNativeInfo(false, string.Empty, false);
}
private void WriteDocumentNativeInfo(bool supportedByViewer, string relativityNativeType, bool hasNative)
{
string str = ConvertToString(supportedByViewer);
string str2 = ConvertToString(hasNative);
OutputFileWriter.OutputNativeFileWriter.Write(str + BulkLoadFileFieldDelimiter);
OutputFileWriter.OutputNativeFileWriter.Write(relativityNativeType + BulkLoadFileFieldDelimiter);
OutputFileWriter.OutputNativeFileWriter.Write(str2 + BulkLoadFileFieldDelimiter);
}
private string ConvertToString(bool booleanValue)
{
if (!booleanValue)
return "0";
return "1";
}
private void WriteDocumentField(ref Encoding chosenEncoding, ArtifactField field, IStreamWriter outputWriter, string delimiter, int artifactTypeID, Encoding extractedTextEncoding)
{
_Closure$__162-1 arg = new _Closure$__162-1(arg);
arg.$VB$Me = this;
arg.$VB$Local_field = field;
arg.$VB$Local_outputWriter = outputWriter;
arg.$VB$Local_extractedTextEncoding = extractedTextEncoding;
if (arg.$VB$Local_field.Type == global::Relativity.DataExchange.Service.FieldType.MultiCode || arg.$VB$Local_field.Type == global::Relativity.DataExchange.Service.FieldType.Code) {
arg.$VB$Local_outputWriter.Write(RuntimeHelpers.GetObjectValue(arg.$VB$Local_field.Value));
arg.$VB$Local_outputWriter.Write(delimiter);
} else if (arg.$VB$Local_field.Type == global::Relativity.DataExchange.Service.FieldType.File && artifactTypeID != 10) {
string[] array = HttpUtility.UrlDecode(arg.$VB$Local_field.ValueAsString).Split(new char[1] {
' '
});
if (array.Length > 1) {
arg.$VB$Local_outputWriter.Write(array[0]);
arg.$VB$Local_outputWriter.Write(delimiter);
arg.$VB$Local_outputWriter.Write(array[1]);
arg.$VB$Local_outputWriter.Write(delimiter);
arg.$VB$Local_outputWriter.Write(array[2]);
arg.$VB$Local_outputWriter.Write(delimiter);
} else {
arg.$VB$Local_outputWriter.Write("");
arg.$VB$Local_outputWriter.Write(delimiter);
arg.$VB$Local_outputWriter.Write("");
arg.$VB$Local_outputWriter.Write(delimiter);
arg.$VB$Local_outputWriter.Write("");
arg.$VB$Local_outputWriter.Write(delimiter);
}
} else if ((arg.$VB$Local_field.Type != global::Relativity.DataExchange.Service.FieldType.File || artifactTypeID != 10) && arg.$VB$Local_field.Category != global::Relativity.DataExchange.Service.FieldCategory.ParentArtifact && arg.$VB$Local_field.ArtifactID > 0) {
if (FieldValueContainsTextFileLocation(arg.$VB$Local_field))
try {
if (Operators.CompareString(arg.$VB$Local_field.ValueAsString, string.Empty, false) != 0) {
_Closure$__162-0 closure$__162-;
closure$__162-.$VB$NonLocal_$VB$Closure_2 = arg;
closure$__162-.$VB$Local_maxRetryAttempts = NumberOfRetries;
closure$__162-.$VB$Local_currentRetryAttempt = 0;
Encoding encoding = CreateWaitAndRetryPolicy().WaitAndRetry(RetryExceptionHelper.CreateRetryPredicate(base.RetryOptions), delegate(int count) {
closure$__162-.$VB$Local_currentRetryAttempt = count;
return TimeSpan.FromSeconds((double)closure$__162-.$VB$NonLocal_$VB$Closure_2.$VB$Me.WaitTimeBetweenRetryAttempts);
}, delegate(Exception exception, TimeSpan span) {
closure$__162-.$VB$NonLocal_$VB$Closure_2.$VB$Me.PublishIoRetryMessage(exception, span, closure$__162-.$VB$Local_currentRetryAttempt, closure$__162-.$VB$Local_maxRetryAttempts);
}, delegate {
Encoding encoding2 = closure$__162-.$VB$NonLocal_$VB$Closure_2.$VB$Local_extractedTextEncoding;
if (closure$__162-.$VB$NonLocal_$VB$Closure_2.$VB$Me.GetFileLength(closure$__162-.$VB$NonLocal_$VB$Closure_2.$VB$Local_field.ValueAsString, false) > closure$__162-.$VB$NonLocal_$VB$Closure_2.$VB$Me.GetMaxExtractedTextLength(encoding2))
throw new ExtractedTextTooLargeException();
if (!closure$__162-.$VB$NonLocal_$VB$Closure_2.$VB$Me.LoadImportedFullTextFromServer) {
Stream stream;
if (!closure$__162-.$VB$NonLocal_$VB$Closure_2.$VB$Me.SkipExtractedTextEncodingCheck) {
DeterminedEncodingStream determinedEncodingStream = Utility.DetectEncoding(closure$__162-.$VB$NonLocal_$VB$Closure_2.$VB$Local_field.ValueAsString, false);
stream = determinedEncodingStream.UnderlyingStream;
Encoding determinedEncoding = determinedEncodingStream.DeterminedEncoding;
if (determinedEncoding != null)
encoding2 = determinedEncoding;
} else
stream = new FileStream(closure$__162-.$VB$NonLocal_$VB$Closure_2.$VB$Local_field.ValueAsString, FileMode.Open, FileAccess.Read);
StreamReader streamReader = new StreamReader(stream, encoding2);
char[] buffer = new char[40001];
int num;
do {
num = streamReader.ReadBlock(buffer, 0, 40000);
if (num > 0) {
closure$__162-.$VB$NonLocal_$VB$Closure_2.$VB$Local_outputWriter.Write(buffer, 0, num);
closure$__162-.$VB$NonLocal_$VB$Closure_2.$VB$Local_outputWriter.Flush();
}
} while (num != 0);
streamReader.Close();
try {
stream.Close();
return encoding2;
} catch (Exception projectError) {
ProjectData.SetProjectError(projectError);
ProjectData.ClearProjectError();
return encoding2;
}
}
if (!closure$__162-.$VB$NonLocal_$VB$Closure_2.$VB$Me.SkipExtractedTextEncodingCheck) {
DeterminedEncodingStream determinedEncodingStream2 = Utility.DetectEncoding(closure$__162-.$VB$NonLocal_$VB$Closure_2.$VB$Local_field.ValueAsString, false);
Stream stream = determinedEncodingStream2.UnderlyingStream;
Field field2 = closure$__162-.$VB$NonLocal_$VB$Closure_2.$VB$Me.get_FullTextField(closure$__162-.$VB$NonLocal_$VB$Closure_2.$VB$Me._settings.ArtifactTypeID);
Encoding objA = (field2 != null && field2.UseUnicodeEncoding) ? Encoding.Unicode : null;
Encoding determinedEncoding2 = determinedEncodingStream2.DeterminedEncoding;
if (!object.Equals(objA, determinedEncoding2))
closure$__162-.$VB$NonLocal_$VB$Closure_2.$VB$Me.WriteWarning("The extracted text file's encoding was not detected to be the same as the extracted text field. The imported data may be incorrectly encoded.");
if (determinedEncoding2 != null)
encoding2 = determinedEncoding2;
try {
stream.Close();
} catch (Exception projectError2) {
ProjectData.SetProjectError(projectError2);
ProjectData.ClearProjectError();
}
}
closure$__162-.$VB$NonLocal_$VB$Closure_2.$VB$Local_outputWriter.Write(RuntimeHelpers.GetObjectValue(closure$__162-.$VB$NonLocal_$VB$Closure_2.$VB$Local_field.Value));
return encoding2;
}, base.CancellationToken);
if (encoding != null)
chosenEncoding = encoding;
}
} catch (FileNotFoundException ex) {
ProjectData.SetProjectError(ex);
FileNotFoundException ex2 = ex;
throw new ExtractedTextFileNotFoundException();
} catch (IOException ex3) {
ProjectData.SetProjectError(ex3);
IOException ex4 = ex3;
if (ExceptionHelper.IsOutOfDiskSpaceException(ex4))
throw;
throw new ImportIOException($"""{arg.$VB$Local_field.DisplayName}""", ex4);
}
else if (arg.$VB$Local_field.Type == global::Relativity.DataExchange.Service.FieldType.Boolean) {
if (Operators.CompareString(arg.$VB$Local_field.ValueAsString, string.Empty, false) != 0) {
if (bool.Parse(arg.$VB$Local_field.ValueAsString))
arg.$VB$Local_outputWriter.Write("1");
else
arg.$VB$Local_outputWriter.Write("0");
}
} else if (arg.$VB$Local_field.Type == global::Relativity.DataExchange.Service.FieldType.Decimal || arg.$VB$Local_field.Type == global::Relativity.DataExchange.Service.FieldType.Currency) {
if (Operators.CompareString(arg.$VB$Local_field.ValueAsString, string.Empty, false) != 0) {
string value = Conversions.ToDecimal(arg.$VB$Local_field.Value).ToString(CultureInfo.InvariantCulture);
arg.$VB$Local_outputWriter.Write(value);
}
} else if (arg.$VB$Local_field.Type == global::Relativity.DataExchange.Service.FieldType.Text || arg.$VB$Local_field.Type == global::Relativity.DataExchange.Service.FieldType.OffTableText) {
if (arg.$VB$Local_field.Value is Stream) {
Stream obj = (Stream)arg.$VB$Local_field.Value;
arg.$VB$Local_outputWriter.Flush();
obj.CopyTo(arg.$VB$Local_outputWriter.BaseStream);
} else
arg.$VB$Local_outputWriter.Write(RuntimeHelpers.GetObjectValue(arg.$VB$Local_field.Value));
} else {
arg.$VB$Local_outputWriter.Write(RuntimeHelpers.GetObjectValue(arg.$VB$Local_field.Value));
}
arg.$VB$Local_outputWriter.Write(delimiter);
}
}
private FieldInfo GetIsSupportedRelativityFileTypeField()
{
Field[] array = this.get_AllFields(_artifactTypeID);
foreach (Field field in array) {
if (Operators.CompareString(field.DisplayName.ToLower(), "supported by viewer", false) == 0)
return FieldDtoToFieldInfo(field);
}
return null;
}
private FieldInfo GetRelativityFileTypeField()
{
Field[] array = this.get_AllFields(_artifactTypeID);
foreach (Field field in array) {
if (Operators.CompareString(field.DisplayName.ToLower(), "relativity native type", false) == 0)
return FieldDtoToFieldInfo(field);
}
return null;
}
private FieldInfo GetHasNativesField()
{
Field[] array = this.get_AllFields(_artifactTypeID);
foreach (Field field in array) {
if (Operators.CompareString(field.DisplayName.ToLower(), "has native", false) == 0)
return FieldDtoToFieldInfo(field);
}
return null;
}
private FieldInfo FieldDtoToFieldInfo(Field input)
{
FieldInfo fieldInfo = new FieldInfo();
fieldInfo.ArtifactID = input.ArtifactID;
fieldInfo.Category = (kCura.EDDS.WebAPI.BulkImportManagerBase.FieldCategory)input.FieldCategoryID;
if (input.CodeTypeID.HasValue)
fieldInfo.CodeTypeID = input.CodeTypeID.Value;
fieldInfo.DisplayName = input.DisplayName;
if (input.MaxLength.HasValue)
fieldInfo.TextLength = input.MaxLength.Value;
fieldInfo.IsUnicodeEnabled = input.UseUnicodeEncoding;
fieldInfo.Type = ConvertFieldTypeEnum(input.FieldTypeID);
fieldInfo.EnableDataGrid = input.EnableDataGrid;
return fieldInfo;
}
private kCura.EDDS.WebAPI.BulkImportManagerBase.FieldType ConvertFieldTypeEnum(int fieldtypeID)
{
global::Relativity.DataExchange.Service.FieldType fieldType = (global::Relativity.DataExchange.Service.FieldType)fieldtypeID;
return (kCura.EDDS.WebAPI.BulkImportManagerBase.FieldType)Conversions.ToInteger(Enum.Parse(typeof(kCura.EDDS.WebAPI.BulkImportManagerBase.FieldType), fieldType.ToString()));
}
private bool IsSupportedRelativityFileType(IFileTypeInfo fileTypeInfo)
{
if (fileTypeInfo != null) {
if (_oixFileLookup == null) {
_oixFileLookup = new HybridDictionary();
int[] array = _documentManager.RetrieveAllUnsupportedOiFileIds();
foreach (int num in array) {
_oixFileLookup.Add(num, num);
}
}
return !_oixFileLookup.Contains(fileTypeInfo.Id);
}
return DisableNativeValidation;
}
protected string PrepareFieldCollectionAndExtractIdentityValue(ArtifactFieldCollection record, int CurrentLineNumber)
{
lock (OutputFileWriter.OutputNativeFileWriter) {
lock (OutputFileWriter.OutputCodeFileWriter) {
lock (OutputFileWriter.OutputObjectFileWriter) {
lock (OutputFileWriter.OutputDataGridFileWriter) {
string text = string.Empty;
ArtifactField artifactField = (_keyFieldID <= 0) ? record.IdentifierField : record[_keyFieldID];
if (artifactField != null && artifactField.Value != null)
text = artifactField.Value.ToString();
if (text == null || Operators.CompareString(text, string.Empty, false) == 0)
throw new IdentityValueNotSetException();
int extractedTextFileCodePageId;
if (_processedKeyFieldValues.ContainsKey(text)) {
string identityValue = text;
extractedTextFileCodePageId = _processedKeyFieldValues[text];
throw new IdentifierOverlapException(identityValue, extractedTextFileCodePageId.ToString());
}
_processedKeyFieldValues.Add(text, CurrentLineNumber);
IEnumerator enumerator = default(IEnumerator);
try {
enumerator = _fieldMap.GetEnumerator();
while (enumerator.MoveNext()) {
LoadFileFieldMap.LoadFileFieldMapItem loadFileFieldMapItem = (LoadFileFieldMap.LoadFileFieldMapItem)enumerator.Current;
if (FirstTimeThrough) {
if (loadFileFieldMapItem.DocumentField == null)
WriteStatusLine(EventType2.Warning, $"""{checked(loadFileFieldMapItem.NativeFileColumnIndex + 1)}""", 0);
if (loadFileFieldMapItem.NativeFileColumnIndex == -1)
WriteStatusLine(EventType2.Warning, $"""{loadFileFieldMapItem.DocumentField.FieldName}""", 0);
}
if (loadFileFieldMapItem.DocumentField != null) {
if (loadFileFieldMapItem.DocumentField.FieldTypeID == 9)
ManageFileField(record[loadFileFieldMapItem.DocumentField.FieldID]);
else {
ArtifactField field = record[loadFileFieldMapItem.DocumentField.FieldID];
int nativeFileColumnIndex = loadFileFieldMapItem.NativeFileColumnIndex;
string identityValue2 = text;
extractedTextFileCodePageId = 0;
SetFieldValue(field, nativeFileColumnIndex, false, identityValue2, ref extractedTextFileCodePageId, loadFileFieldMapItem.DocumentField.ImportBehavior);
}
}
}
} finally {
if (enumerator is IDisposable)
(enumerator as IDisposable).Dispose();
}
IEnumerator enumerator2 = default(IEnumerator);
try {
enumerator2 = UnmappedRelationalFields.GetEnumerator();
while (enumerator2.MoveNext()) {
Field field2 = (Field)enumerator2.Current;
int? nullable = (int?)field2.ImportBehavior;
if ((nullable.HasValue ? new bool?(nullable.GetValueOrDefault() == 1) : null).GetValueOrDefault()) {
ArtifactField artifactField2 = new ArtifactField(field2);
artifactField2.Value = text;
ArtifactField field3 = artifactField2;
string identityValue3 = text;
extractedTextFileCodePageId = 0;
SetFieldValue(field3, -1, false, identityValue3, ref extractedTextFileCodePageId, field2.ImportBehavior);
}
}
} finally {
if (enumerator2 is IDisposable)
(enumerator2 as IDisposable).Dispose();
}
FirstTimeThrough = false;
return text;
}
}
}
}
}
private void ManageFileField(ArtifactField fileField)
{
if (fileField != null && fileField.Value != null && Operators.CompareString(fileField.Value.ToString(), string.Empty, false) != 0) {
string text = fileField.Value.ToString();
if (!GetFileExists(text))
throw new FileNotFoundException($"""{text}""");
long fileLength = GetFileLength(text, true);
string text2 = Path.GetFileName(text).Replace(" ", "_");
string str;
if (Operators.CompareString(base.FileTapiBridge.TargetFolderName, "", false) == 0)
str = text;
else {
string str2 = base.FileTapiBridge.AddPath(text, Guid.NewGuid().ToString(), CurrentLineNumber);
str = base.FileTapiBridge.TargetPath + base.FileTapiBridge.TargetFolderName + "\\" + str2;
}
str = HttpUtility.UrlEncode(str);
fileField.Value = string.Format("{1}{0}{2}{0}{3}", ' ', text2, fileLength, str);
}
}
private void WriteTapiProgressMessage(string message, int lineNumber)
{
message = GetLineMessage(message, lineNumber);
int fileTapiProgressCount = base.FileTapiProgressCount;
OnStatusMessage(new StatusEventArgs(EventType2.Progress, fileTapiProgressCount, RecordCount, message, base.CurrentStatisticsSnapshot, base.Statistics));
}
protected void WriteStatusLine(EventType2 et, string line)
{
WriteStatusLine(et, line, CurrentLineNumber);
}
private void WriteStatusLine(EventType2 et, string line, int lineNumber)
{
int num = lineNumber;
if (num != 0)
num = checked(num + Offset);
if (num < 0)
num = 0;
line = GetLineMessage(line, lineNumber);
OnStatusMessage(new StatusEventArgs(et, num, RecordCount, line, base.CurrentStatisticsSnapshot, base.Statistics));
}
private void WriteFatalError(int lineNumber, Exception exception)
{
_artifactReader.OnFatalErrorState();
StopImport(false);
OnFatalError($"""{lineNumber.ToString()}", exception, _runId);
}
private void WriteError(int currentLineNumber, string line)
{
ErrorBeforeMassImportArgs toWrite = new ErrorBeforeMassImportArgs(currentLineNumber);
prePushErrorWriter.WriteErrorMessage(toWrite);
Hashtable row = new Hashtable {
{
"Message",
line
},
{
"Line Number",
currentLineNumber
},
{
"Identifier",
_artifactReader.SourceIdentifierValue()
}
};
RaiseReportError(row, _artifactReader.SourceIdentifierValue(), "client");
WriteStatusLine(EventType2.Error, line);
}
private unsafe void RaiseReportError(Hashtable row, string identifier, string type)
{
ref int errorCount;
*(ref errorCount = ref _errorCount) = checked(errorCount + 1);
if (_errorCount < MaxNumberOfErrorsInGrid)
OnReportErrorEvent(row);
else if (_errorCount == MaxNumberOfErrorsInGrid) {
Hashtable hashtable = new Hashtable();
hashtable.Add("Message", "Maximum number of errors for display reached. Export errors to view full list.");
OnReportErrorEvent(hashtable);
}
ErrorDuringMassImportArgs toWrite = new ErrorDuringMassImportArgs(row["Line Number"].ToString(), row["Message"].ToString(), identifier, type);
errorMessageFileWriter.WriteErrorMessage(toWrite);
}
protected void WriteWarning(string line)
{
WriteStatusLine(EventType2.Warning, line);
}
private void WriteEndImport(string line)
{
if (base.ShouldImport)
WriteStatusLine(EventType2.End, line);
else if (base.CancellationToken.IsCancellationRequested) {
WriteStatusLine(EventType2.Status, "Job has been finalized.", 0);
} else {
WriteStatusLine(EventType2.End, line, base.FileTapiProgressCount);
}
}
protected override void OnWriteStatusMessage(EventType2 eventType, string message, int progressLineNumber, int physicalLineNumber)
{
switch (eventType) {
case EventType2.End:
case EventType2.ResetStartTime:
case EventType2.Count:
case EventType2.ResetProgress:
break;
case EventType2.Error:
WriteError(progressLineNumber, message);
break;
case EventType2.Status:
case EventType2.Progress:
case EventType2.Warning:
case EventType2.Statistics:
WriteStatusLine(eventType, message, progressLineNumber);
break;
}
}
protected override void OnWriteFatalError(Exception exception)
{
WriteFatalError(CurrentLineNumber, exception);
base.OnWriteFatalError(exception);
}
private void LegacyUploader_UploadStatusEvent(string s)
{
WriteStatusLine(EventType2.Status, s);
}
protected override void OnTapiClientChanged()
{
base.OnTapiClientChanged();
PublishUploadModeChangeEvent(_settings.CopyFilesToDocumentRepository && _settings.LoadNativeFiles && _settings.NativeFilePathColumn != null);
}
protected virtual void _processContext_HaltProcessEvent(object sender, CancellationRequestEventArgs e)
{
if (Operators.CompareString(e.ProcessId.ToString(), _processId.ToString(), false) == 0) {
_cancelledByUser = e.RequestByUser;
StopImport(_cancelledByUser);
}
}
protected override void OnStopImport()
{
if (_artifactReader != null)
_artifactReader.Halt();
WriteStatusLine(EventType2.Progress, $"""{base.TotalTransferredFilesCount}""", checked(base.TotalTransferredFilesCount + 1));
WriteStatusLine(EventType2.Status, "Finalizing job...", 0);
}
protected virtual void _processContext_ExportServerErrors(object sender, ExportErrorEventArgs e)
{
errorMessageFileWriter.ReleaseLock();
prePushErrorWriter.ReleaseLock();
string errorMessageFileLocation = errorMessageFileWriter.FileCreated ? errorMessageFileWriter.FilePath : string.Empty;
string prePushErrorLineNumbersFileName = prePushErrorWriter.FileCreated ? prePushErrorWriter.FilePath : string.Empty;
_errorLinesFileLocation = _artifactReader.ManageErrorRecords(errorMessageFileLocation, prePushErrorLineNumbersFileName);
string text = _filePath;
string str;
if (text.IndexOf(".") != -1) {
str = text.Substring(text.LastIndexOf("."));
text = text.Substring(0, text.LastIndexOf("."));
} else
str = ".txt";
if (text.IndexOf("\\") != -1)
text = text.Substring(checked(text.LastIndexOf("\\") + 1));
string str2 = e.Path + text;
DateTime now = DateTime.Now;
string destFileName = str2 + "_ErrorLines_" + Conversions.ToString(now.Ticks) + str;
string destFileName2 = str2 + "_ErrorReport_" + Conversions.ToString(now.Ticks) + ".csv";
if (_errorLinesFileLocation != null && Operators.CompareString(_errorLinesFileLocation, string.Empty, false) != 0 && GetFileExists(_errorLinesFileLocation))
CopyFile(_errorLinesFileLocation, destFileName, true);
string filePath = errorMessageFileWriter.FilePath;
errorMessageFileWriter.Dispose();
errorMessageFileWriter = new ErrorMessageWriter<ErrorDuringMassImportArgs>();
CopyFile(filePath, destFileName2, true);
}
private void _processContext_ExportErrorReportEvent(object sender, ExportErrorEventArgs e)
{
if (!errorMessageFileWriter.FileCreated)
System.IO.File.CreateText(e.Path).Close();
else
CopyFile(errorMessageFileWriter.FilePath, e.Path, true);
}
private void _processContext_ExportErrorFileEvent(object sender, ExportErrorEventArgs e)
{
errorMessageFileWriter.ReleaseLock();
prePushErrorWriter.ReleaseLock();
if (errorMessageFileWriter.FileCreated) {
string prePushErrorLineNumbersFileName = prePushErrorWriter.FileCreated ? prePushErrorWriter.FilePath : string.Empty;
if (_errorLinesFileLocation == null || Operators.CompareString(_errorLinesFileLocation, "", false) == 0 || !GetFileExists(_errorLinesFileLocation))
_errorLinesFileLocation = _artifactReader.ManageErrorRecords(errorMessageFileWriter.FilePath, prePushErrorLineNumbersFileName);
if (_errorLinesFileLocation != null)
CopyFile(_errorLinesFileLocation, e.Path, true);
}
}
private void _artifactReader_DataSourcePrep(DataSourcePrepEventArgs e)
{
OnDataSourcePrepEvent(e);
}
private void _artifactReader_StatusMessage(string message)
{
OnStatusMessage(new StatusEventArgs(EventType2.Status, _artifactReader.CurrentLineNumber, RecordCount, message, false, base.CurrentStatisticsSnapshot, base.Statistics));
}
private void _artifactReader_FieldMapped(string sourceField, string workspaceField)
{
OnFieldMapped(sourceField, workspaceField);
}
private void IoWarningHandler(object sender, IoWarningEventArgs e)
{
IoWarningEventArgs args = new IoWarningEventArgs(e.Message, e.CurrentLineNumber);
PublishIoWarningEvent(args);
}
private unsafe void ManageErrors(int artifactTypeID)
{
if (BulkImportManager.NativeRunHasErrors(_caseInfo.ArtifactID, _runId)) {
GenericCsvReader2 genericCsvReader = null;
ErrorFileService errorFileService = null;
try {
global::Relativity.DataExchange.Service.ErrorFileKey errorFileKey = BulkImportManager.GenerateNonImageErrorFiles(_caseInfo.ArtifactID, _runId, artifactTypeID, true, _keyFieldID);
WriteStatusLine(EventType2.Status, "Retrieving errors from server");
errorFileService = new ErrorFileService((NetworkCredential)BulkImportManager.Credentials, _caseInfo.DownloadHandlerURL, BulkImportManager.CookieContainer, GetCorrelationId);
errorFileService.UploadStatusEvent += LegacyUploader_UploadStatusEvent;
string tempFileName = TempFileBuilder.GetTempFileName("rel-errors");
genericCsvReader = AttemptErrorFileDownload(errorFileService, tempFileName, errorFileKey.LogKey, _caseInfo);
checked {
if (genericCsvReader == null)
OnFatalError("There was an error while attempting to retrieve the errors from the server.", new Exception("There was an error while attempting to retrieve the errors from the server."), _runId);
else {
genericCsvReader.Context.IoWarningEvent += IoWarningHandler;
for (string[] array = genericCsvReader.ReadLine(); array != null; array = genericCsvReader.ReadLine()) {
ref int errorCount;
*(ref errorCount = ref _errorCount) = errorCount + 1;
Hashtable hashtable = new Hashtable();
hashtable.Add("Message", array[1]);
hashtable.Add("Identifier", array[2]);
hashtable.Add("Line Number", int.Parse(array[0]));
RaiseReportError(hashtable, array[2], "server");
OnStatusMessage(new StatusEventArgs(EventType2.Error, int.Parse(array[0]) - 1, RecordCount, "[Line " + array[0] + "]" + array[1], base.CurrentStatisticsSnapshot, base.Statistics));
}
genericCsvReader.Context.IoWarningEvent -= IoWarningHandler;
}
errorFileService.UploadStatusEvent -= LegacyUploader_UploadStatusEvent;
errorFileKey = null;
}
} catch (Exception ex) {
ProjectData.SetProjectError(ex);
Exception ex2 = ex;
try {
if (errorFileService != null)
errorFileService.UploadStatusEvent -= LegacyUploader_UploadStatusEvent;
genericCsvReader.Close();
genericCsvReader.Context.IoWarningEvent -= IoWarningHandler;
} catch (Exception projectError) {
ProjectData.SetProjectError(projectError);
ProjectData.ClearProjectError();
}
throw;
}
}
}
private GenericCsvReader2 AttemptErrorFileDownload(ErrorFileService errorFileService, string errorFileOutputPath, string logKey, CaseInfo caseInfo)
{
int num = 3;
GenericCsvReader2 genericCsvReader = null;
while (num > 0) {
errorFileService.DownloadErrorFile(errorFileOutputPath, logKey, caseInfo, false);
genericCsvReader = new GenericCsvReader2(errorFileOutputPath, Encoding.UTF8, true);
if (genericCsvReader.Peek() != -1)
break;
num = checked(num - 1);
genericCsvReader.Close();
genericCsvReader = null;
}
errorFileService.RemoveErrorFile(logKey, caseInfo);
return genericCsvReader;
}
private void _processContext_ParentFormClosingEvent(object sender, ParentFormClosingEventArgs e)
{
if (Operators.CompareString(e.ProcessId.ToString(), _processId.ToString(), false) == 0)
CleanupTempTables();
}
protected void CleanupTempTables()
{
if (_runId != null && Operators.CompareString(_runId, "", false) != 0)
try {
BulkImportManager.DisposeTempTables(_caseInfo.ArtifactID, _runId);
} catch (Exception ex) {
ProjectData.SetProjectError(ex);
Exception exception = ex;
LogWarning(exception, "Failed to drop the {RunId} SQL temp tables.", _runId);
ProjectData.ClearProjectError();
}
}
protected override Base GetSingleCodeValidator()
{
return new SingleImporter(_settings.CaseInfo, _codeManager);
}
protected override IArtifactReader GetArtifactReader()
{
return new LoadFileReader(_settings, false, GetCorrelationId, _executionSource);
}
protected override string GetCorrelationId()
{
if (string.IsNullOrEmpty(_runId))
return CorrelationIdFunc?.Invoke();
return _runId;
}
protected void OnFatalError(string message, Exception ex, string runID)
{
FatalErrorEventEvent?.Invoke(message, ex, runID);
}
protected void OnStatusMessage(StatusEventArgs args)
{
StatusMessageEvent?.Invoke(args);
}
protected void OnEndFileImport(string runID)
{
EndFileImportEvent?.Invoke(runID);
}
protected void OnStartFileImport()
{
StartFileImportEvent?.Invoke();
}
protected void OnDataSourcePrepEvent(DataSourcePrepEventArgs args)
{
DataSourcePrepEventEvent?.Invoke(args);
}
protected void OnReportErrorEvent(IDictionary row)
{
ReportErrorEventEvent?.Invoke(row);
}
protected void OnFieldMapped(string sourceField, string workspaceField)
{
FieldMappedEvent?.Invoke(sourceField, workspaceField);
}
public virtual void Dispose()
{
errorMessageFileWriter?.Dispose();
prePushErrorWriter?.Dispose();
}
void IDisposable.Dispose()
{
this.Dispose();
}
[CompilerGenerated]
private bool _Lambda$__136-0(Exception exception)
{
FileTypeIdentifyException ex = exception as FileTypeIdentifyException;
if (ex == null)
return RetryExceptionHelper.IsRetryable(exception, base.RetryOptions);
if (ex.Error != FileTypeIdentifyError.Permissions)
return true;
return base.RetryOptions.HasFlag(RetryOptions.Permissions);
}
[CompilerGenerated]
[CompilerGenerated]
private void $VB$ClosureStub_OnBatchCompleted_MyBase(BatchInformation batchInformation)
{
base.OnBatchCompleted(batchInformation);
}
}
}