Merge branch 'release/7.0.0'

Author: Jericho

Source/HttpMultipartParser.UnitTests/ParserScenarios/InvalidPart.cs

@@ -0,0 +1,73 @@
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Xunit;
+
+namespace HttpMultipartParser.UnitTests.ParserScenarios
+{
+	public class InvalidPart
+	{
+		// For details see: https://github.com/Http-Multipart-Data-Parser/Http-Multipart-Data-Parser/issues/110
+		// This data is considered invalid because it contains nothing but empty lines
+		private static readonly string _testData = @"--KoZIhvcNAQcB
+
+--KoZIhvcNAQcB--";
+
+		private static readonly TestData _testCase = new TestData(
+			_testData,
+			Enumerable.Empty<ParameterPart>().ToList(),
+			Enumerable.Empty<FilePart>().ToList()
+		);
+
+		public InvalidPart()
+		{
+			foreach (var filePart in _testCase.ExpectedFileData)
+			{
+				filePart.Data.Position = 0;
+			}
+		}
+
+		[Fact]
+		public void Exception_is_thrown_when_attempting_to_parse()
+		{
+			// The default behavior is to throw an exception when the form contains an invalid section.
+			using (Stream stream = TestUtil.StringToStream(_testCase.Request, Encoding.UTF8))
+			{
+				Assert.Throws<MultipartParseException>(() => MultipartFormDataParser.Parse(stream));
+			}
+		}
+
+		[Fact]
+		public async Task Exception_is_thrown_when_attempting_to_parse_async()
+		{
+			// The default behavior is to throw an exception when the form contains an invalid section.
+			using (Stream stream = TestUtil.StringToStream(_testCase.Request, Encoding.UTF8))
+			{
+				await Assert.ThrowsAsync<MultipartParseException>(() => MultipartFormDataParser.ParseAsync(stream)).ConfigureAwait(false);
+			}
+		}
+
+		[Fact]
+		public void Invalid_part_is_ignored()
+		{
+			using (Stream stream = TestUtil.StringToStream(_testCase.Request, Encoding.UTF8))
+			{
+				var parser = MultipartFormDataParser.Parse(stream, ignoreInvalidParts: true);
+				Assert.Equal(0, parser.Files.Count);
+				Assert.Equal(0, parser.Parameters.Count);
+			}
+		}
+
+		[Fact]
+		public async Task Invalid_part_is_ignored_async()
+		{
+			using (Stream stream = TestUtil.StringToStream(_testCase.Request, Encoding.UTF8))
+			{
+				var parser = await MultipartFormDataParser.ParseAsync(stream, ignoreInvalidParts: true).ConfigureAwait(false);
+				Assert.Equal(0, parser.Files.Count);
+				Assert.Equal(0, parser.Parameters.Count);
+			}
+		}
+	}
+}

Source/HttpMultipartParser/MultipartFormDataParser.cs

@@ -146,12 +146,15 @@ private MultipartFormDataParser()
 		/// <param name="binaryMimeTypes">
 		///     List of mimetypes that should be detected as file.
 		/// </param>
+		/// <param name="ignoreInvalidParts">
+		///     By default the parser will throw an exception if it encounters an invalid part. Set this to true to ignore invalid parts.
+		/// </param>
 		/// <returns>
 		///     A new instance of the <see cref="MultipartFormDataParser"/> class.
 		/// </returns>
-		public static MultipartFormDataParser Parse(Stream stream, Encoding encoding, int binaryBufferSize = DefaultBufferSize, string[] binaryMimeTypes = null)
+		public static MultipartFormDataParser Parse(Stream stream, Encoding encoding, int binaryBufferSize = DefaultBufferSize, string[] binaryMimeTypes = null, bool ignoreInvalidParts = false)
 		{
-			return Parse(stream, null, encoding, binaryBufferSize, binaryMimeTypes);
+			return Parse(stream, null, encoding, binaryBufferSize, binaryMimeTypes, ignoreInvalidParts);
 		}
 
 		/// <summary>
@@ -175,13 +178,16 @@ public static MultipartFormDataParser Parse(Stream stream, Encoding encoding, in
 		/// <param name="binaryMimeTypes">
 		///     List of mimetypes that should be detected as file.
 		/// </param>
+		/// <param name="ignoreInvalidParts">
+		///     By default the parser will throw an exception if it encounters an invalid part. Set this to true to ignore invalid parts.
+		/// </param>
 		/// <returns>
 		///     A new instance of the <see cref="MultipartFormDataParser"/> class.
 		/// </returns>
-		public static MultipartFormDataParser Parse(Stream stream, string boundary = null, Encoding encoding = null, int binaryBufferSize = DefaultBufferSize, string[] binaryMimeTypes = null)
+		public static MultipartFormDataParser Parse(Stream stream, string boundary = null, Encoding encoding = null, int binaryBufferSize = DefaultBufferSize, string[] binaryMimeTypes = null, bool ignoreInvalidParts = false)
 		{
 			var parser = new MultipartFormDataParser();
-			parser.ParseStream(stream, boundary, encoding, binaryBufferSize, binaryMimeTypes);
+			parser.ParseStream(stream, boundary, encoding, binaryBufferSize, binaryMimeTypes, ignoreInvalidParts);
 			return parser;
 		}
 
@@ -202,12 +208,15 @@ public static MultipartFormDataParser Parse(Stream stream, string boundary = nul
 		/// <param name="binaryMimeTypes">
 		///     List of mimetypes that should be detected as file.
 		/// </param>
+		/// <param name="ignoreInvalidParts">
+		///     By default the parser will throw an exception if it encounters an invalid part. Set this to true to ignore invalid parts.
+		/// </param>
 		/// <returns>
 		///     A new instance of the <see cref="MultipartFormDataParser"/> class.
 		/// </returns>
-		public static Task<MultipartFormDataParser> ParseAsync(Stream stream, Encoding encoding, int binaryBufferSize = DefaultBufferSize, string[] binaryMimeTypes = null)
+		public static Task<MultipartFormDataParser> ParseAsync(Stream stream, Encoding encoding, int binaryBufferSize = DefaultBufferSize, string[] binaryMimeTypes = null, bool ignoreInvalidParts = false)
 		{
-			return ParseAsync(stream, null, encoding);
+			return ParseAsync(stream, null, encoding, DefaultBufferSize, null, ignoreInvalidParts);
 		}
 
 		/// <summary>
@@ -231,13 +240,16 @@ public static Task<MultipartFormDataParser> ParseAsync(Stream stream, Encoding e
 		/// <param name="binaryMimeTypes">
 		///     List of mimetypes that should be detected as file.
 		/// </param>
+		/// <param name="ignoreInvalidParts">
+		///     By default the parser will throw an exception if it encounters an invalid part. Set this to true to ignore invalid parts.
+		/// </param>
 		/// <returns>
 		///     A new instance of the <see cref="MultipartFormDataParser"/> class.
 		/// </returns>
-		public static async Task<MultipartFormDataParser> ParseAsync(Stream stream, string boundary = null, Encoding encoding = null, int binaryBufferSize = DefaultBufferSize, string[] binaryMimeTypes = null)
+		public static async Task<MultipartFormDataParser> ParseAsync(Stream stream, string boundary = null, Encoding encoding = null, int binaryBufferSize = DefaultBufferSize, string[] binaryMimeTypes = null, bool ignoreInvalidParts = false)
 		{
 			var parser = new MultipartFormDataParser();
-			await parser.ParseStreamAsync(stream, boundary, encoding, binaryBufferSize, binaryMimeTypes).ConfigureAwait(false);
+			await parser.ParseStreamAsync(stream, boundary, encoding, binaryBufferSize, binaryMimeTypes, ignoreInvalidParts).ConfigureAwait(false);
 			return parser;
 		}
 
@@ -265,9 +277,12 @@ public static async Task<MultipartFormDataParser> ParseAsync(Stream stream, stri
 		/// <param name="binaryMimeTypes">
 		///     List of mimetypes that should be detected as file.
 		/// </param>
-		private void ParseStream(Stream stream, string boundary, Encoding encoding, int binaryBufferSize, string[] binaryMimeTypes)
+		/// <param name="ignoreInvalidParts">
+		///     By default the parser will throw an exception if it encounters an invalid part. Set this to true to ignore invalid parts.
+		/// </param>
+		private void ParseStream(Stream stream, string boundary, Encoding encoding, int binaryBufferSize, string[] binaryMimeTypes, bool ignoreInvalidParts)
 		{
-			var streamingParser = new StreamingMultipartFormDataParser(stream, boundary, encoding ?? Encoding.UTF8, binaryBufferSize, binaryMimeTypes);
+			var streamingParser = new StreamingMultipartFormDataParser(stream, boundary, encoding ?? Encoding.UTF8, binaryBufferSize, binaryMimeTypes, ignoreInvalidParts);
 			streamingParser.ParameterHandler += parameterPart => _parameters.Add(parameterPart);
 
 			streamingParser.FileHandler += (name, fileName, type, disposition, buffer, bytes, partNumber, additionalProperties) =>
@@ -310,9 +325,12 @@ private void ParseStream(Stream stream, string boundary, Encoding encoding, int
 		/// <param name="binaryMimeTypes">
 		///     List of mimetypes that should be detected as file.
 		/// </param>
-		private async Task ParseStreamAsync(Stream stream, string boundary, Encoding encoding, int binaryBufferSize, string[] binaryMimeTypes)
+		/// <param name="ignoreInvalidParts">
+		///     By default the parser will throw an exception if it encounters an invalid part. Set this to true to ignore invalid parts.
+		/// </param>
+		private async Task ParseStreamAsync(Stream stream, string boundary, Encoding encoding, int binaryBufferSize, string[] binaryMimeTypes, bool ignoreInvalidParts)
 		{
-			var streamingParser = new StreamingMultipartFormDataParser(stream, boundary, encoding ?? Encoding.UTF8, binaryBufferSize, binaryMimeTypes);
+			var streamingParser = new StreamingMultipartFormDataParser(stream, boundary, encoding ?? Encoding.UTF8, binaryBufferSize, binaryMimeTypes, ignoreInvalidParts);
 			streamingParser.ParameterHandler += parameterPart => _parameters.Add(parameterPart);
 
 			streamingParser.FileHandler += (name, fileName, type, disposition, buffer, bytes, partNumber, additionalProperties) =>

Source/HttpMultipartParser/StreamingMultipartFormDataParser.cs

@@ -55,20 +55,30 @@ public class StreamingMultipartFormDataParser : IStreamingMultipartFormDataParse
 		/// </remarks>
 		private const int DefaultBufferSize = 4096;
 
+		/// <summary>
+		/// The mimetypes that are considered a file by default.
+		/// </summary>
+		private static readonly string[] DefaultBinaryMimeTypes = { "application/octet-stream" };
+
 		#endregion
 
 		#region Fields
 
 		/// <summary>
 		///     List of mimetypes that should be detected as file.
 		/// </summary>
-		private readonly string[] binaryMimeTypes = { "application/octet-stream" };
+		private readonly string[] binaryMimeTypes;
 
 		/// <summary>
 		///     The stream we are parsing.
 		/// </summary>
 		private readonly Stream stream;
 
+		/// <summary>
+		///     Determines if we should throw an exception when we enconter an invalid part or ignore it.
+		/// </summary>
+		private readonly bool ignoreInvalidParts;
+
 		/// <summary>
 		///     The boundary of the multipart message  as a string.
 		/// </summary>
@@ -118,8 +128,11 @@ public class StreamingMultipartFormDataParser : IStreamingMultipartFormDataParse
 		/// <param name="binaryMimeTypes">
 		///     List of mimetypes that should be detected as file.
 		/// </param>
-		public StreamingMultipartFormDataParser(Stream stream, Encoding encoding, int binaryBufferSize = DefaultBufferSize, string[] binaryMimeTypes = null)
-			: this(stream, null, encoding, binaryBufferSize, binaryMimeTypes)
+		/// <param name="ignoreInvalidParts">
+		///     By default the parser will throw an exception if it encounters an invalid part. set this to true to ignore invalid parts.
+		/// </param>
+		public StreamingMultipartFormDataParser(Stream stream, Encoding encoding, int binaryBufferSize = DefaultBufferSize, string[] binaryMimeTypes = null, bool ignoreInvalidParts = false)
+			: this(stream, null, encoding, binaryBufferSize, binaryMimeTypes, ignoreInvalidParts)
 		{
 		}
 
@@ -144,7 +157,10 @@ public StreamingMultipartFormDataParser(Stream stream, Encoding encoding, int bi
 		/// <param name="binaryMimeTypes">
 		///     List of mimetypes that should be detected as file.
 		/// </param>
-		public StreamingMultipartFormDataParser(Stream stream, string boundary = null, Encoding encoding = null, int binaryBufferSize = DefaultBufferSize, string[] binaryMimeTypes = null)
+		/// <param name="ignoreInvalidParts">
+		///     By default the parser will throw an exception if it encounters an invalid part. set this to true to ignore invalid parts.
+		/// </param>
+		public StreamingMultipartFormDataParser(Stream stream, string boundary = null, Encoding encoding = null, int binaryBufferSize = DefaultBufferSize, string[] binaryMimeTypes = null, bool ignoreInvalidParts = false)
 		{
 			if (stream == null || stream == Stream.Null) { throw new ArgumentNullException(nameof(stream)); }
 
@@ -153,10 +169,8 @@ public StreamingMultipartFormDataParser(Stream stream, string boundary = null, E
 			Encoding = encoding ?? Encoding.UTF8;
 			BinaryBufferSize = binaryBufferSize;
 			readEndBoundary = false;
-			if (binaryMimeTypes != null)
-			{
-				this.binaryMimeTypes = binaryMimeTypes;
-			}
+			this.binaryMimeTypes = binaryMimeTypes ?? DefaultBinaryMimeTypes;
+			this.ignoreInvalidParts = ignoreInvalidParts;
 		}
 
 		#endregion
@@ -346,24 +360,43 @@ private static async Task<string> DetectBoundaryAsync(RebufferableBinaryReader r
 		}
 
 		/// <summary>
-		/// Use a few assumptions to determine if a section contains a file or a "data" parameter.
+		/// Use a few assumptions to determine if a section contains a file.
 		/// </summary>
 		/// <param name="parameters">The section parameters.</param>
 		/// <returns>true if the section contains a file, false otherwise.</returns>
-		private bool IsFilePart(IDictionary<string, string> parameters)
+		private bool IsFilePart(IDictionary<string, string> parameters!!)
 		{
+			// A section without any parameter is invalid. It is very likely to contain just a bunch of blank lines.
+			if (parameters.Count == 0) return false;
+
 			// If a section contains filename, then it's a file.
-			if (parameters.ContainsKey("filename")) return true;
+			else if (parameters.ContainsKey("filename")) return true;
 
 			// Check if mimetype is a binary file
-			else if (parameters.ContainsKey("content-type") &&
-					 binaryMimeTypes.Contains(parameters["content-type"])) return true;
+			else if (parameters.ContainsKey("content-type") && binaryMimeTypes.Contains(parameters["content-type"])) return true;
 
 			// If the section is missing the filename and the name, then it's a file.
 			// For example, images in an mjpeg stream have neither a name nor a filename.
 			else if (!parameters.ContainsKey("name")) return true;
 
-			// In all other cases, we assume it's a "data" parameter.
+			// Otherwise this section does not contain a file.
+			return false;
+		}
+
+		/// <summary>
+		/// Use a few assumptions to determine if a section contains a "data" parameter.
+		/// </summary>
+		/// <param name="parameters">The section parameters.</param>
+		/// <returns>true if the section contains a data parameter, false otherwise.</returns>
+		private bool IsParameterPart(IDictionary<string, string> parameters!!)
+		{
+			// A section without any parameter is invalid. It is very likely to contain just a bunch of blank lines.
+			if (parameters.Count == 0) return false;
+
+			// A data parameter MUST have a name.
+			else if (parameters.ContainsKey("name")) return true;
+
+			// Otherwise this section does not contain a data parameter.
 			return false;
 		}
 
@@ -950,6 +983,58 @@ private async Task ParseParameterPartAsync(Dictionary<string, string> parameters
 			ParameterHandler(part);
 		}
 
+		/// <summary>
+		///     Skip a section of the stream.
+		///     This is used when a section is deemed to be invalid and the developer has requested to ignore invalid parts.
+		/// </summary>
+		/// <param name="reader">
+		///     The StreamReader to read the data from.
+		/// </param>
+		/// <exception cref="MultipartParseException">
+		///     thrown if unexpected data is found such as running out of stream before hitting the boundary.
+		/// </exception>
+		private void SkipPart(RebufferableBinaryReader reader)
+		{
+			// Our job is to consume the lines in this section and discard them
+			string line = reader.ReadLine();
+			while (line != boundary && line != endBoundary)
+			{
+				if (line == null) throw new MultipartParseException("Unexpected end of stream. Is there an end boundary?");
+				line = reader.ReadLine();
+			}
+
+			if (line == endBoundary) readEndBoundary = true;
+		}
+
+		/// <summary>
+		///     Asynchronously skip a section of the stream.
+		///     This is used when a section is deemed to be invalid and the developer has requested to ignore invalid parts.
+		/// </summary>
+		/// <param name="reader">
+		///     The StreamReader to read the data from.
+		/// </param>
+		/// <param name="cancellationToken">
+		///     The cancellation token.
+		/// </param>
+		/// <returns>
+		///     The asynchronous task.
+		/// </returns>
+		/// <exception cref="MultipartParseException">
+		///     thrown if unexpected data is found such as running out of stream before hitting the boundary.
+		/// </exception>
+		private async Task SkipPartAsync(RebufferableBinaryReader reader, CancellationToken cancellationToken = default)
+		{
+			// Our job is to consume the lines in this section and discard them
+			string line = await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false);
+			while (line != boundary && line != endBoundary)
+			{
+				if (line == null) throw new MultipartParseException("Unexpected end of stream. Is there an end boundary?");
+				line = await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false);
+			}
+
+			if (line == endBoundary) readEndBoundary = true;
+		}
+
 		/// <summary>
 		///     Parses the header of the next section of the multipart stream and
 		///     determines if it contains file data or parameter data.
@@ -1029,10 +1114,18 @@ private void ParseSection(RebufferableBinaryReader reader)
 			{
 				ParseFilePart(parameters, reader);
 			}
-			else
+			else if (IsParameterPart(parameters))
 			{
 				ParseParameterPart(parameters, reader);
 			}
+			else if (ignoreInvalidParts)
+			{
+				SkipPart(reader);
+			}
+			else
+			{
+				throw new MultipartParseException("Unable to determine the section type. Some possible reasons include: section is malformed, required parameters such as 'name', 'content-type' or 'filename' are missing, section contains nothing but empty lines.");
+			}
 		}
 
 		/// <summary>
@@ -1120,10 +1213,18 @@ private async Task ParseSectionAsync(RebufferableBinaryReader reader, Cancellati
 			{
 				await ParseFilePartAsync(parameters, reader, cancellationToken).ConfigureAwait(false);
 			}
-			else
+			else if (IsParameterPart(parameters))
 			{
 				await ParseParameterPartAsync(parameters, reader, cancellationToken).ConfigureAwait(false);
 			}
+			else if (ignoreInvalidParts)
+			{
+				await SkipPartAsync(reader).ConfigureAwait(false);
+			}
+			else
+			{
+				throw new MultipartParseException("Unable to determine the section type. Some possible reasons include: section is malformed, required parameters such as 'name', 'content-type' or 'filename' are missing, section contains nothing but empty lines.");
+			}
 		}
 
 		/// <summary>