DelimitedFileImporter2
Represents an abstract representation of a file importer that operates over delimited files.
using Microsoft.VisualBasic;
using Microsoft.VisualBasic.CompilerServices;
using Relativity.DataExchange.Io;
using Relativity.DataExchange.Resources;
using Relativity.Logging;
using System;
using System.Globalization;
using System.IO;
using System.Text;
using System.Threading;
namespace Relativity.DataExchange.Data
{
public abstract class DelimitedFileImporter2 : IoReporter
{
public const int MaxColumnCountForLine = 10000;
private const char UnboundedChar = '';
private const char UnspecifiedNewlineProxyChar = '';
private const int EofChar = -1;
private string newlineProxyString;
public int CurrentLineNumber { get; set; }
protected char Bound { get; }
protected long CharacterPosition { get; set; }
protected char Delimiter { get; }
protected virtual bool HasReachedEof => Peek() == -1;
protected DelimitedMode Mode {
get {
if (Bound != '')
return DelimitedMode.Bounded;
return DelimitedMode.Simple;
}
}
protected char NewlineProxy { get; }
protected StreamReader Reader { get; set; }
protected virtual bool UseConcordanceStyleBoundStart => true;
protected TrimOption TrimOption { get; set; }
protected DelimitedFileImporter2(char delimiter, bool retry)
: this(delimiter, '', '', new IoReporterContext {
RetryOptions = (retry ? RetryOptions.Io : RetryOptions.None)
}, new NullLogger(), CancellationToken.None)
{
}
protected DelimitedFileImporter2(char delimiter, char bound, char newlineProxy, bool retry)
: this(delimiter, bound, newlineProxy, new IoReporterContext {
RetryOptions = (retry ? RetryOptions.Io : RetryOptions.None)
}, new NullLogger(), CancellationToken.None)
{
}
protected DelimitedFileImporter2(char delimiter, IoReporterContext context, ILog logger, CancellationToken token)
: this(delimiter, '', '', context, logger, token)
{
}
protected DelimitedFileImporter2(char delimiter, char bound, char newlineProxy, IoReporterContext context, ILog logger, CancellationToken token)
: base(context, logger, token)
{
Delimiter = delimiter;
Bound = bound;
NewlineProxy = newlineProxy;
}
public static string ValidateStringForVarChar(string value, int column, int fieldLength, int currentLineNumber, string displayName)
{
if (value != null && value.Length > fieldLength)
throw new StringImporterException(currentLineNumber, column, value.Length, fieldLength, displayName);
return NullableTypesHelper.ToString(value);
}
public void Close()
{
if (Reader != null) {
Reader.Close();
Reader = null;
}
}
public abstract object ReadFile(string path);
public bool GetBoolean(string value)
{
if (string.Compare(value, "true", StringComparison.OrdinalIgnoreCase) == 0)
return true;
if (string.Compare(value, "false", StringComparison.OrdinalIgnoreCase) == 0)
return false;
if (int.TryParse(value, out int result))
return result != 0;
return false;
}
public decimal GetDecimal(string value, int column)
{
try {
decimal num = decimal.Parse(value, CultureInfo.InvariantCulture);
if (Conversions.ToString(decimal.Truncate(num)).Length > 15)
throw new FormatException("The decimal value " + value + " exceeds the max length.");
return num;
} catch (Exception innerException) {
throw new DecimalImporterException(CurrentLineNumber, column, innerException);
}
}
public bool? GetNullableBoolean(string value, int column)
{
try {
return NullableTypesHelper.GetNullableBoolean(value);
} catch (Exception innerException) {
throw new BooleanImporterException(CurrentLineNumber, column, innerException);
}
}
public DateTime? GetNullableDateTime(string value, int column)
{
try {
return NullableTypesHelper.GetNullableDateTime(value);
} catch (Exception innerException) {
throw new DateImporterException(CurrentLineNumber, column, innerException);
}
}
public decimal? GetNullableDecimal(string value, int column)
{
try {
decimal? result = ParseNullableDecimal(value);
if (result.HasValue && Conversions.ToString(decimal.Truncate(result.Value)).Length > 15)
throw new FormatException("The decimal value " + value + " exceeds the max length.");
return result;
} catch (Exception innerException) {
throw new DecimalImporterException(CurrentLineNumber, column, innerException);
}
}
public string GetNullableFixedString(string value, int column, int maxLength, string displayName)
{
if (value != null && value.Length > maxLength)
throw new StringImporterException(CurrentLineNumber, column, value.Length, maxLength, displayName);
return NullableTypesHelper.ToString(value);
}
public int? GetNullableInteger(string value, int column)
{
try {
return NullableTypesHelper.ToNullableInt32(value);
} catch (Exception innerException) {
throw new IntegerImporterException(CurrentLineNumber, column, innerException);
}
}
public virtual int Peek()
{
return GetChar(false);
}
public void ResetLineCounter()
{
CurrentLineNumber = 0;
}
public void Rewind()
{
Reader.BaseStream.Seek(0, SeekOrigin.Begin);
CurrentLineNumber = 0;
CharacterPosition = 0;
}
protected void AdvanceLine()
{
switch (Mode) {
case DelimitedMode.Bounded:
GetLineBounded(false, 2147483647, 2147483647);
break;
case DelimitedMode.Simple:
GetLineSimple(false);
break;
}
}
protected string[] GetLine()
{
return GetLine(2147483647);
}
protected string[] GetLine(int maximumFieldLength)
{
string[] result = null;
switch (Mode) {
case DelimitedMode.Bounded:
result = GetLineBounded(true, maximumFieldLength, 10000);
break;
case DelimitedMode.Simple:
result = GetLineSimple(true);
break;
}
return result;
}
protected int GetChar(bool advance)
{
int num = base.IoErrorNumberOfRetries;
bool flag = false;
while (num > 0 && !base.CancellationToken.IsCancellationRequested) {
try {
num--;
if (flag) {
ReInitializeReader();
flag = false;
}
if (!advance)
return Reader.Peek();
int result = Reader.Read();
CharacterPosition++;
return result;
} catch (Exception ex) {
if (num == 0 || base.Context.RetryOptions == RetryOptions.None)
throw;
int num2 = 0;
if (num < base.IoErrorNumberOfRetries - 1)
num2 = base.IoErrorWaitTimeInSeconds;
IoWarningEventArgs args = new IoWarningEventArgs(num2, ex, CurrentLineNumber);
PublishWarningMessage(args);
if ((Reader.BaseStream is FileStream || Reader.BaseStream is RestartableFileStream) && ex is IOException && ex.Message.ToLowerInvariant().Contains("network"))
flag = true;
if (num2 > 0)
Thread.CurrentThread.Join(1000 * num2);
}
}
return 0;
}
protected virtual decimal? ParseNullableDecimal(string value)
{
return NullableTypesHelper.ToNullableDecimal(value, NumberStyles.AllowLeadingWhite | NumberStyles.AllowTrailingWhite | NumberStyles.AllowLeadingSign | NumberStyles.AllowTrailingSign | NumberStyles.AllowDecimalPoint | NumberStyles.AllowThousands | NumberStyles.AllowCurrencySymbol);
}
private void Append(StringBuilder sb, char value, long startPosition, int maxCharacterLength, ref bool hasAlertedError)
{
if (ValidateAppend(startPosition, maxCharacterLength, ref hasAlertedError))
sb.Append(value);
}
private void Append(StringBuilder sb, string value, long startPosition, int maxCharacterLength, ref bool hasAlertedError)
{
if (ValidateAppend(startPosition, maxCharacterLength, ref hasAlertedError))
sb.Append(value);
}
private bool ValidateAppend(long startPosition, int maxCharacterLength, ref bool hasAlertedError)
{
if (checked(CharacterPosition - startPosition) > maxCharacterLength) {
if (!hasAlertedError) {
hasAlertedError = true;
string message = "Contents of cell has exceeded maximum length of " + Conversions.ToString(maxCharacterLength) + " (character " + Conversions.ToString(CharacterPosition) + ")";
PublishWarningMessage(new IoWarningEventArgs(message, CurrentLineNumber));
}
return false;
}
return true;
}
private string ConvertNewLine(StringBuilder input)
{
return ConvertNewLine(input.ToString());
}
private string ConvertNewLine(string input)
{
if (NewlineProxy != '') {
if (string.IsNullOrEmpty(newlineProxyString))
newlineProxyString = Conversions.ToString(NewlineProxy);
input = input.Replace("\r\n", newlineProxyString);
input = input.Replace("\r", newlineProxyString);
input = input.Replace("\n", newlineProxyString);
}
switch (TrimOption) {
case TrimOption.None:
return input;
case TrimOption.Leading:
return input.TrimStart(' ', '\t');
case TrimOption.Trailing:
return input.TrimEnd(' ', '\t');
case TrimOption.Both:
return input.Trim(' ', '\t');
default:
return null;
}
}
private string GetBoundedFieldValue(ref bool hasHitEndOfLine, int column, int maximumFieldLength)
{
StringBuilder stringBuilder = new StringBuilder();
long characterPosition = CharacterPosition;
bool hasAlertedError = false;
if (Peek() == Bound) {
Microsoft.VisualBasic.Strings.ChrW(Read());
if (Microsoft.VisualBasic.Strings.ChrW(Peek()) == Delimiter || Peek() == -1 || Peek() == 13) {
if (Peek() != 13) {
Read();
return string.Empty;
}
Read();
if (Peek() == 10) {
Read();
hasHitEndOfLine = true;
return string.Empty;
}
Append(stringBuilder, Conversions.ToString(Bound) + "\r", characterPosition, maximumFieldLength, ref hasAlertedError);
} else {
Append(stringBuilder, Bound, characterPosition, maximumFieldLength, ref hasAlertedError);
if (UseConcordanceStyleBoundStart)
Read();
}
}
while (Peek() != -1) {
char c = Microsoft.VisualBasic.Strings.ChrW(Read());
if (c == Bound) {
if (TrimOption == TrimOption.Both || TrimOption == TrimOption.Trailing) {
char[] array = new char[2] {
' ',
'\t'
};
while (Array.IndexOf(array, Microsoft.VisualBasic.Strings.ChrW(Peek())) != -1) {
Read();
}
}
int num = Peek();
if (num == Delimiter) {
Read();
return ConvertNewLine(stringBuilder);
}
switch (num) {
case -1:
return ConvertNewLine(stringBuilder);
case 13: {
char value = Microsoft.VisualBasic.Strings.ChrW(Read());
if (Peek() == 10) {
Read();
hasHitEndOfLine = true;
return ConvertNewLine(stringBuilder);
}
Append(stringBuilder, value, characterPosition, maximumFieldLength, ref hasAlertedError);
break;
}
default:
if (num == Bound) {
Append(stringBuilder, Bound, characterPosition, maximumFieldLength, ref hasAlertedError);
Read();
}
break;
}
} else
Append(stringBuilder, c, characterPosition, maximumFieldLength, ref hasAlertedError);
}
try {
return stringBuilder.ToString();
} catch (Exception innerException) {
throw new CellImporterException(CurrentLineNumber, column, innerException);
}
}
private string[] GetLineBounded(bool saveData, int maximumFieldLength, int maximumLineLength)
{
ConditionalArrayList conditionalArrayList = new ConditionalArrayList(saveData);
ConditionalArrayList conditionalArrayList2 = conditionalArrayList;
bool hasHitEndOfLine = false;
while (Peek() != -1 && !hasHitEndOfLine) {
int charCode = Read();
if (conditionalArrayList2.Count > maximumLineLength)
conditionalArrayList2 = new ConditionalArrayList(false);
while ((Microsoft.VisualBasic.Strings.ChrW(charCode) == ' ' || Microsoft.VisualBasic.Strings.ChrW(charCode) == '\t') && (TrimOption == TrimOption.Both || TrimOption == TrimOption.Leading)) {
charCode = Read();
}
char c = Microsoft.VisualBasic.Strings.ChrW(charCode);
if (c == Bound)
conditionalArrayList2.Add(GetBoundedFieldValue(ref hasHitEndOfLine, conditionalArrayList2.Count, maximumFieldLength));
else if (c == Delimiter) {
conditionalArrayList2.Add(string.Empty);
if (Peek() == -1)
conditionalArrayList2.Add(string.Empty);
} else if (c == '\r') {
if (Peek() == 10) {
conditionalArrayList2.Add(string.Empty);
Read();
hasHitEndOfLine = true;
} else
conditionalArrayList2.Add(Conversions.ToString(Microsoft.VisualBasic.Strings.ChrW(charCode)) + GetSimpleDelimitedValue(ref hasHitEndOfLine, maximumFieldLength));
} else if ((Microsoft.VisualBasic.Strings.ChrW(charCode) != ' ' && Microsoft.VisualBasic.Strings.ChrW(charCode) != '\t') || (TrimOption != TrimOption.Both && TrimOption != TrimOption.Leading)) {
conditionalArrayList2.Add(Conversions.ToString(Microsoft.VisualBasic.Strings.ChrW(charCode)) + GetSimpleDelimitedValue(ref hasHitEndOfLine, maximumFieldLength));
}
}
CurrentLineNumber++;
if (conditionalArrayList2 != conditionalArrayList) {
IoWarningEventArgs args = new IoWarningEventArgs("This line's column length has exceeded the maximum defined column length of " + Conversions.ToString(10000) + ". Remaining columns in line truncated", CurrentLineNumber);
PublishWarningMessage(args);
}
return (string[])conditionalArrayList.ToArray(typeof(string));
}
private string[] GetLineSimple(bool saveData)
{
ConditionalStringBuilder conditionalStringBuilder = new ConditionalStringBuilder(saveData);
while (Peek() != -1) {
int num = Read();
switch (num) {
case 13:
break;
case 10:
goto IL_0035;
default:
goto IL_0046;
}
if (Read() == 10)
break;
continue;
IL_0035:
conditionalStringBuilder.Append(NewlineProxy);
continue;
IL_0046:
conditionalStringBuilder.Append(Microsoft.VisualBasic.Strings.ChrW(num));
}
string text = conditionalStringBuilder.ToString();
CurrentLineNumber++;
return text.Split(new char[1] {
Delimiter
});
}
private string GetSimpleDelimitedValue(ref bool hasHitEndOfLine, int maximumCharacters)
{
StringBuilder stringBuilder = new StringBuilder();
long characterPosition = CharacterPosition;
bool hasAlertedError = false;
while (Peek() != -1) {
int num = Read();
int num2 = num;
if (num2 == Delimiter)
return stringBuilder.ToString();
if (num2 == 13) {
if (Peek() == 10) {
Read();
hasHitEndOfLine = true;
return stringBuilder.ToString();
}
Append(stringBuilder, Microsoft.VisualBasic.Strings.ChrW(num), characterPosition, maximumCharacters, ref hasAlertedError);
} else
Append(stringBuilder, Microsoft.VisualBasic.Strings.ChrW(num), characterPosition, maximumCharacters, ref hasAlertedError);
}
return stringBuilder.ToString();
}
private int Read()
{
return GetChar(true);
}
private void ReInitializeReader()
{
RestartableFileStream restartableFileStream = Reader.BaseStream as RestartableFileStream;
if (restartableFileStream != null) {
PublishWarningMessage(new IoWarningEventArgs("Re-initializing stream for broken network connection", CurrentLineNumber));
restartableFileStream.Restart();
} else {
FileStream fileStream = Reader.BaseStream as FileStream;
if (fileStream == null)
throw new InvalidOperationException(Relativity.DataExchange.Resources.Strings.ReinitializeReaderNotFileStreamError);
PublishWarningMessage(new IoWarningEventArgs("Re-initializing reader for broken network connection", CurrentLineNumber));
string name = fileStream.Name;
long position = fileStream.Position;
Encoding currentEncoding = Reader.CurrentEncoding;
Reader.Dispose();
Stream stream = new RestartableFileStream(name);
try {
Reader = new StreamReader(stream, currentEncoding, position == 0);
} catch {
stream.Dispose();
throw;
}
long num = Math.Max(CharacterPosition / 10, 100000);
for (int i = 0; i <= CharacterPosition; i++) {
if (i % num == 0)
PublishWarningMessage(new IoWarningEventArgs($"{i}""{CharacterPosition}""", CurrentLineNumber));
Reader.Read();
}
PublishWarningMessage(new IoWarningEventArgs($"{CharacterPosition}""", CurrentLineNumber));
}
}
}
}