Add stop_after_first_success parameter and fix upload params · sdaves/NpgsqlRest@3e28204 · GitHub
Skip to content

Commit 3e28204

Browse files
vbilopavvbilopav
committed
Add stop_after_first_success parameter and fix upload params
1 parent 49b58df commit 3e28204

23 files changed

Lines changed: 379 additions & 233 deletions

‎BenchmarkTests/BenchmarkTests.csproj‎

Lines changed: 2 additions & 2 deletions

‎NpgsqlRest/NpgsqlRestMiddleware.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313

1414
using NpgsqlRest.Auth;
1515
using NpgsqlRest.UploadHandlers;
16+
using NpgsqlRest.UploadHandlers.Handlers;
1617

1718
namespace NpgsqlRest;
1819

‎NpgsqlRest/NpgsqlRestOptions.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
using Npgsql;
44
using NpgsqlRest.Defaults;
55
using NpgsqlRest.UploadHandlers;
6+
using NpgsqlRest.UploadHandlers.Handlers;
67

78
namespace NpgsqlRest;
89

‎NpgsqlRest/UploadHandlers/FileCheckExtensions.cs‎

Lines changed: 0 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -11,42 +11,6 @@ public static class FileCheckExtensions
1111
public const string TestBufferSizeParam = "test_buffer_size";
1212
public const string NonPrintableThresholdParam = "non_printable_threshold";
1313

14-
public static bool CheckMimeTypes(this string contentType, string[]? includedMimeTypePatterns, string[]? excludedMimeTypePatterns)
15-
{
16-
// File must match AT LEAST ONE included pattern
17-
if (includedMimeTypePatterns is not null && includedMimeTypePatterns.Length > 0)
18-
{
19-
bool matchesAny = false;
20-
for (int j = 0; j < includedMimeTypePatterns.Length; j++)
21-
{
22-
if (Parser.IsPatternMatch(contentType, includedMimeTypePatterns[j]))
23-
{
24-
matchesAny = true;
25-
break;
26-
}
27-
}
28-
29-
if (!matchesAny)
30-
{
31-
return false;
32-
}
33-
}
34-
35-
// File must NOT match ANY excluded patterns
36-
if (excludedMimeTypePatterns is not null)
37-
{
38-
for (int j = 0; j < excludedMimeTypePatterns.Length; j++)
39-
{
40-
if (Parser.IsPatternMatch(contentType, excludedMimeTypePatterns[j]))
41-
{
42-
return false;
43-
}
44-
}
45-
}
46-
47-
return true;
48-
}
49-
5014
public static async Task<UploadFileStatus> CheckTextContentStatus(
5115
this IFormFile formFile,
5216
int testBufferSize = 4096,

‎NpgsqlRest/UploadHandlers/Handlers/BaseUploadHandler.cs‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
namespace NpgsqlRest.UploadHandlers.Handlers;
2+
3+
public abstract class BaseUploadHandler
4+
{
5+
protected HashSet<string> _skipFileNames = new(StringComparer.OrdinalIgnoreCase);
6+
7+
protected string? _type = default;
8+
protected bool CheckMimeTypes(string contentType)
9+
{
10+
// File must match AT LEAST ONE included pattern
11+
if (_includedMimeTypePatterns is not null && _includedMimeTypePatterns.Length > 0)
12+
{
13+
bool matchesAny = false;
14+
for (int j = 0; j < _includedMimeTypePatterns.Length; j++)
15+
{
16+
if (Parser.IsPatternMatch(contentType, _includedMimeTypePatterns[j]))
17+
{
18+
matchesAny = true;
19+
break;
20+
}
21+
}
22+
23+
if (!matchesAny)
24+
{
25+
return false;
26+
}
27+
}
28+
29+
// File must NOT match ANY excluded patterns
30+
if (_excludedMimeTypePatterns is not null)
31+
{
32+
for (int j = 0; j < _excludedMimeTypePatterns.Length; j++)
33+
{
34+
if (Parser.IsPatternMatch(contentType, _excludedMimeTypePatterns[j]))
35+
{
36+
return false;
37+
}
38+
}
39+
}
40+
41+
return true;
42+
}
43+
44+
protected bool TryGetParam(Dictionary<string, string> parameters, string key, out string value)
45+
{
46+
if (parameters.TryGetValue(key, out var val))
47+
{
48+
value = val;
49+
return true;
50+
}
51+
if (parameters.TryGetValue(string.Concat(_type, "_", key), out val))
52+
{
53+
value = val;
54+
return true;
55+
}
56+
value = default!;
57+
return false;
58+
}
59+
60+
protected abstract IEnumerable<string> GetParameters();
61+
62+
protected string[]? _includedMimeTypePatterns = default;
63+
protected string[]? _excludedMimeTypePatterns = default;
64+
protected int _bufferSize = default;
65+
protected bool _stopAfterFirstSuccess = default;
66+
67+
public void ParseSharedParameters(NpgsqlRestUploadOptions options, Dictionary<string, string>? parameters)
68+
{
69+
_includedMimeTypePatterns = options.DefaultUploadHandlerOptions.IncludedMimeTypePatterns;
70+
_excludedMimeTypePatterns = options.DefaultUploadHandlerOptions.ExcludedMimeTypePatterns;
71+
_bufferSize = options.DefaultUploadHandlerOptions.BufferSize;
72+
_stopAfterFirstSuccess = options.DefaultUploadHandlerOptions.StopAfterFirstSuccess;
73+
74+
if (parameters is not null)
75+
{
76+
if (TryGetParam(parameters, IncludedMimeTypeParam, out var includedMimeTypeStr) && includedMimeTypeStr is not null)
77+
{
78+
_includedMimeTypePatterns = includedMimeTypeStr.SplitParameter();
79+
}
80+
if (TryGetParam(parameters, ExcludedMimeTypeParam, out var excludedMimeTypeStr) && excludedMimeTypeStr is not null)
81+
{
82+
_excludedMimeTypePatterns = excludedMimeTypeStr.SplitParameter();
83+
}
84+
if (TryGetParam(parameters, BufferSizeParam, out var bufferSizeStr) && int.TryParse(bufferSizeStr, out var bufferSizeParsed))
85+
{
86+
_bufferSize = bufferSizeParsed;
87+
}
88+
if (TryGetParam(parameters, StopAfterFirstParam, out var stopAfterFirstSuccessStr) && bool.TryParse(stopAfterFirstSuccessStr, out var stopAfterFirstSuccessParsed))
89+
{
90+
_stopAfterFirstSuccess = stopAfterFirstSuccessParsed;
91+
}
92+
}
93+
}
94+
95+
public IUploadHandler SetType(string type)
96+
{
97+
_type = type;
98+
return (this as IUploadHandler)!;
99+
}
100+
101+
public IEnumerable<string> Parameters
102+
{
103+
get
104+
{
105+
yield return StopAfterFirstParam;
106+
foreach (var param in GetParameters())
107+
{
108+
yield return param;
109+
}
110+
foreach (var param in GetParameters())
111+
{
112+
yield return string.Concat(_type, "_", param);
113+
}
114+
}
115+
}
116+
117+
public const string StopAfterFirstParam = "stop_after_first_success";
118+
public const string IncludedMimeTypeParam = "included_mime_types";
119+
public const string ExcludedMimeTypeParam = "excluded_mime_types";
120+
public const string BufferSizeParam = "buffer_size";
121+
122+
public bool StopAfterFirst => _stopAfterFirstSuccess;
123+
124+
public void SetSkipFileNames(HashSet<string> skipFileNames)
125+
{
126+
_skipFileNames = skipFileNames;
127+
}
128+
129+
public HashSet<string> GetSkipFileNames => _skipFileNames;
130+
}

‎NpgsqlRest/UploadHandlers/Handlers/CsvUploadHandler.cs‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
namespace NpgsqlRest.UploadHandlers.Handlers;
88

9-
public class CsvUploadHandler(NpgsqlRestUploadOptions options, ILogger? logger) : UploadHandler, IUploadHandler
9+
public class CsvUploadHandler(NpgsqlRestUploadOptions options, ILogger? logger) : BaseUploadHandler, IUploadHandler
1010
{
1111
private const string CheckFileParam = "check_csv";
1212
private const string DelimitersParam = "delimiters";
@@ -31,8 +31,6 @@ protected override IEnumerable<string> GetParameters()
3131

3232
public async Task<string> UploadAsync(NpgsqlConnection connection, HttpContext context, Dictionary<string, string>? parameters)
3333
{
34-
var (includedMimeTypePatterns, excludedMimeTypePatterns, _) = ParseSharedParameters(options, parameters);
35-
3634
bool checkFileStatus = options.DefaultUploadHandlerOptions.CsvUploadCheckFileStatus;
3735
int testBufferSize = options.DefaultUploadHandlerOptions.TextTestBufferSize;
3836
int nonPrintableThreshold = options.DefaultUploadHandlerOptions.TextNonPrintableThreshold;
@@ -75,10 +73,8 @@ public async Task<string> UploadAsync(NpgsqlConnection connection, HttpContext c
7573

7674
if (options.LogUploadParameters is true)
7775
{
78-
#pragma warning disable CA2253 // Named placeholders should not be numeric values
79-
logger?.LogInformation("Upload for {0}: includedMimeTypePatterns={1}, excludedMimeTypePatterns={2}, checkFileStatus={3}, testBufferSize={4}, nonPrintableThreshold={5}, delimiters={6}, hasFieldsEnclosedInQuotes={7}, setWhiteSpaceToNull={8}, rowCommand={9}",
80-
_type, includedMimeTypePatterns, excludedMimeTypePatterns, checkFileStatus, testBufferSize, nonPrintableThreshold, delimiters, hasFieldsEnclosedInQuotes, setWhiteSpaceToNull, rowCommand);
81-
#pragma warning disable CA2253 // Named placeholders should not be numeric values
76+
logger?.LogInformation("Upload for {_type}: includedMimeTypePatterns={includedMimeTypePatterns}, excludedMimeTypePatterns={excludedMimeTypePatterns}, checkFileStatus={checkFileStatus}, testBufferSize={testBufferSize}, nonPrintableThreshold={nonPrintableThreshold}, delimiters={delimiters}, hasFieldsEnclosedInQuotes={hasFieldsEnclosedInQuotes}, setWhiteSpaceToNull={setWhiteSpaceToNull}, rowCommand={rowCommand}",
77+
_type, _includedMimeTypePatterns, _excludedMimeTypePatterns, checkFileStatus, testBufferSize, nonPrintableThreshold, delimiters, hasFieldsEnclosedInQuotes, setWhiteSpaceToNull, rowCommand);
8278
}
8379

8480
string[] delimitersArr = [.. delimiters.Select(c => c.ToString())];
@@ -118,7 +114,11 @@ public async Task<string> UploadAsync(NpgsqlConnection connection, HttpContext c
118114
fileJson.Append(formFile.Length);
119115

120116
UploadFileStatus status = UploadFileStatus.Ok;
121-
if (formFile.ContentType.CheckMimeTypes(includedMimeTypePatterns, excludedMimeTypePatterns) is false)
117+
if (_stopAfterFirstSuccess is true && _skipFileNames.Contains(formFile.FileName, StringComparer.OrdinalIgnoreCase))
118+
{
119+
status = UploadFileStatus.Ignored;
120+
}
121+
if (status == UploadFileStatus.Ok && this.CheckMimeTypes(formFile.ContentType) is false)
122122
{
123123
status = UploadFileStatus.InvalidMimeType;
124124
}
@@ -138,6 +138,10 @@ public async Task<string> UploadAsync(NpgsqlConnection connection, HttpContext c
138138
fileId++;
139139
continue;
140140
}
141+
if (_stopAfterFirstSuccess is true)
142+
{
143+
_skipFileNames.Add(formFile.FileName);
144+
}
141145

142146
using var fileStream = formFile.OpenReadStream();
143147
using var streamReader = new StreamReader(fileStream);

‎NpgsqlRest/UploadHandlers/Handlers/DefaultUploadHandler.cs‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,9 @@
33

44
namespace NpgsqlRest.UploadHandlers.Handlers;
55

6-
public class DefaultUploadHandler(params IUploadHandler[] handlers) : IUploadHandler
6+
public class DefaultUploadHandler(NpgsqlRestUploadOptions options, IUploadHandler[] handlers) : IUploadHandler
77
{
8+
private readonly NpgsqlRestUploadOptions _options = options;
89
private readonly IUploadHandler[] _handlers = handlers;
910
private readonly bool _requiresTransaction = handlers.Any(h => h.RequiresTransaction);
1011

@@ -23,7 +24,18 @@ public async Task<string> UploadAsync(NpgsqlConnection connection, HttpContext c
2324
StringBuilder result = new(100);
2425
for (int i = 0; i < _handlers.Length; i++)
2526
{
26-
var segment = await _handlers[i].UploadAsync(connection, context, parameters);
27+
var handler = _handlers[i];
28+
if (handler is BaseUploadHandler baseHandler)
29+
{
30+
baseHandler.ParseSharedParameters(_options, parameters);
31+
if (i > 0 && baseHandler.StopAfterFirst is true && _handlers[i - 1] is BaseUploadHandler prevHandler)
32+
{
33+
baseHandler.SetSkipFileNames(prevHandler.GetSkipFileNames);
34+
}
35+
}
36+
37+
var segment = await handler.UploadAsync(connection, context, parameters);
38+
2739
if (i == 0 && _handlers.Length == 1)
2840
{
2941
result.Append(segment);

‎NpgsqlRest/UploadHandlers/Handlers/FileSystemUploadHandler.cs‎

Lines changed: 19 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
namespace NpgsqlRest.UploadHandlers.Handlers;
66

7-
public class FileSystemUploadHandler(NpgsqlRestUploadOptions options, ILogger? logger) : UploadHandler, IUploadHandler
7+
public class FileSystemUploadHandler(NpgsqlRestUploadOptions options, ILogger? logger) : BaseUploadHandler, IUploadHandler
88
{
99
private string[]? _uploadedFiles = null;
1010

@@ -16,7 +16,7 @@ protected override IEnumerable<string> GetParameters()
1616
{
1717
yield return IncludedMimeTypeParam;
1818
yield return ExcludedMimeTypeParam;
19-
yield return BufferSize;
19+
yield return BufferSizeParam;
2020
yield return PathParam;
2121
yield return FileParam;
2222
yield return UniqueNameParam;
@@ -31,14 +31,12 @@ protected override IEnumerable<string> GetParameters()
3131

3232
public async Task<string> UploadAsync(NpgsqlConnection connection, HttpContext context, Dictionary<string, string>? parameters)
3333
{
34-
var (includedMimeTypePatterns, excludedMimeTypePatterns, bufferSize) = ParseSharedParameters(options, parameters);
35-
36-
var basePath = options.DefaultUploadHandlerOptions.FileSystemHandlerPath;
37-
var useUniqueFileName = options.DefaultUploadHandlerOptions.FileSystemHandlerUseUniqueFileName;
34+
var basePath = options.DefaultUploadHandlerOptions.FileSystemPath;
35+
var useUniqueFileName = options.DefaultUploadHandlerOptions.FileSystemUseUniqueFileName;
3836
string? newFileName = null;
39-
bool createPathIfNotExists = options.DefaultUploadHandlerOptions.FileSystemHandlerCreatePathIfNotExists;
40-
bool checkText = false;
41-
bool checkImage = false;
37+
bool createPathIfNotExists = options.DefaultUploadHandlerOptions.FileSystemCreatePathIfNotExists;
38+
bool checkText = options.DefaultUploadHandlerOptions.FileSystemCheckText;
39+
bool checkImage = options.DefaultUploadHandlerOptions.FileSystemCheckImage;
4240
int testBufferSize = options.DefaultUploadHandlerOptions.TextTestBufferSize;
4341
int nonPrintableThreshold = options.DefaultUploadHandlerOptions.TextNonPrintableThreshold;
4442

@@ -93,10 +91,8 @@ public async Task<string> UploadAsync(NpgsqlConnection connection, HttpContext c
9391

9492
if (options.LogUploadParameters is true)
9593
{
96-
#pragma warning disable CA2253 // Named placeholders should not be numeric values
97-
logger?.LogInformation("Upload for {0}: includedMimeTypePatterns={1}, excludedMimeTypePatterns={2}, bufferSize={3}, basePath={4}, useUniqueFileName={5}, newFileName={6}, createPathIfNotExists={7}, checkText={8}, checkImage={9}, allowedImage={10}, testBufferSize={11}, nonPrintableThreshold={12}",
98-
_type, includedMimeTypePatterns, excludedMimeTypePatterns, bufferSize, basePath, useUniqueFileName, newFileName, createPathIfNotExists, checkText, checkImage, allowedImage, testBufferSize, nonPrintableThreshold);
99-
#pragma warning disable CA2253 // Named placeholders should not be numeric values
94+
logger?.LogInformation("Upload for {_type}: includedMimeTypePatterns={includedMimeTypePatterns}, excludedMimeTypePatterns={excludedMimeTypePatterns}, bufferSize={bufferSize}, basePath={basePath}, useUniqueFileName={useUniqueFileName}, newFileName={newFileName}, createPathIfNotExists={createPathIfNotExists}, checkText={checkText}, checkImage={checkImage}, allowedImage={allowedImage}, testBufferSize={testBufferSize}, nonPrintableThreshold={nonPrintableThreshold}",
95+
_type, _includedMimeTypePatterns, _excludedMimeTypePatterns, _bufferSize, basePath, useUniqueFileName, newFileName, createPathIfNotExists, checkText, checkImage, allowedImage, testBufferSize, nonPrintableThreshold);
10096
}
10197

10298
if (createPathIfNotExists is true && Directory.Exists(basePath) is false)
@@ -147,7 +143,11 @@ public async Task<string> UploadAsync(NpgsqlConnection connection, HttpContext c
147143
result.Append(SerializeString(currentFilePath));
148144

149145
UploadFileStatus status = UploadFileStatus.Ok;
150-
if (formFile.ContentType.CheckMimeTypes(includedMimeTypePatterns, excludedMimeTypePatterns) is false)
146+
if (_stopAfterFirstSuccess is true && _skipFileNames.Contains(formFile.FileName, StringComparer.OrdinalIgnoreCase))
147+
{
148+
status = UploadFileStatus.Ignored;
149+
}
150+
if (status == UploadFileStatus.Ok && this.CheckMimeTypes(formFile.ContentType) is false)
151151
{
152152
status = UploadFileStatus.InvalidMimeType;
153153
}
@@ -176,10 +176,14 @@ public async Task<string> UploadAsync(NpgsqlConnection connection, HttpContext c
176176
fileId++;
177177
continue;
178178
}
179+
if (_stopAfterFirstSuccess is true)
180+
{
181+
_skipFileNames.Add(formFile.FileName);
182+
}
179183

180184
using (var fileStream = new FileStream(currentFilePath, FileMode.Create))
181185
{
182-
byte[] buffer = new byte[bufferSize];
186+
byte[] buffer = new byte[_bufferSize];
183187
int bytesRead;
184188
using var sourceStream = formFile.OpenReadStream();
185189

NpgsqlRest/UploadHandlers/IUploadHandler.cs renamed to NpgsqlRest/UploadHandlers/Handlers/IUploadHandler.cs

Lines changed: 1 addition & 1 deletion

0 commit comments

Comments
 (0)