...
 
Commits (13)
......@@ -23,7 +23,7 @@ update-test:
script:
- cp src/PommaLabs.MimeTypes/MimeTypeMap.cs MimeTypeMap.cs.bak
- cp test/PommaLabs.MimeTypes.UnitTests/MimeTypeMapTests.cs MimeTypeMapTests.cs.bak
- pwsh update-maps.ps1
- pwsh update-map.ps1
- diff src/PommaLabs.MimeTypes/MimeTypeMap.cs MimeTypeMap.cs.bak
- diff test/PommaLabs.MimeTypes.UnitTests/MimeTypeMapTests.cs MimeTypeMapTests.cs.bak
......
......@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Changed
- Library now targets .NET Standard 2.0, .NET Framework 4.6.1 and 4.7.2.
- Mime types are now fetched from [mime-types-data project](https://github.com/mime-types/mime-types-data).
## [1.1.0] - 2020-03-22
......
......@@ -87,7 +87,8 @@ Console.WriteLine(MimeTypeMap.GetMimeType(MimeTypeMap.Extensions._3gpp)); // "vi
```
Extensions starting with a digit are prefixed with a `_` character,
since C# field names cannot start with a digit.
since C# field names cannot start with a digit. For the same reason,
invalid leading characters are replaced with a `_` character.
### Compatibility
......@@ -139,4 +140,5 @@ This project was inspired by the following open source projects:
Moreover, this project heavily relies on the following open source projects:
- [DotLiquid](https://github.com/dotliquid/dotliquid), used to process Liquid templates.
\ No newline at end of file
- [DotLiquid](https://github.com/dotliquid/dotliquid), used to process Liquid templates.
- [mime-types-data](https://github.com/mime-types/mime-types-data), used to retrive data for the map.
\ No newline at end of file
......@@ -31,12 +31,13 @@ namespace PommaLabs.MimeTypes.Generator
/// </summary>
internal static class CustomFilters
{
private static readonly char[] s_splitChars = new[] { '-', '+', '.', '_' };
private static readonly char[] s_splitChars = new[] { '-', '+', '.', '_', '@' };
public static string Sharpify(string w)
{
var c = w[0];
w = CapitalizeParts(w);
return char.IsDigit(w[0]) ? $"_{w}" : w;
return (char.IsDigit(c) || s_splitChars.Contains(c)) ? $"_{w}" : w;
}
private static string Capitalize(string w)
......
// File name: MimeTypeMapTests.cs
// File name: MimeTypeMapTests.Generated.cs
//
// Author(s): Alessio Parma <[email protected]>
//
......@@ -27,14 +27,11 @@ using NUnit.Framework;
// WARNING: This file was auto generated by DotLiquid. DON'T modify it manually.
namespace PommaLabs.MimeTypes.UnitTests
{
public sealed class MimeTypeMapTests
public sealed partial class MimeTypeMapTests
{
private const string DefaultExtension = ".bin";
private const string DefaultMimeType = "application/octet-stream";
{% for gkv in mime_types -%}
{% for kv in gkv.Value -%}
[TestCase(MimeTypeMap.{{ gkv.Key | sharpify }}.{{ kv.Key | sharpify }}, ".{{ kv.Value }}")]
[TestCase(MimeTypeMap.{{ gkv.Key | sharpify }}.{{ kv.Key | sharpify }}, ".{{ kv.Value | first }}")]
{% endfor -%}
{% endfor -%}
public void GetExtension_ExistingMimeTypeShouldBeMatchedToCorrectExtension(string mimeType, string extension)
......@@ -43,19 +40,6 @@ namespace PommaLabs.MimeTypes.UnitTests
Assert.That(MimeTypeMap.GetExtensions(mimeType), Contains.Item(extension));
}
[Test]
public void GetExtension_MissingMimeTypeShouldBeMatchedToBinExtensionWhenThrowExceptionIsFalse()
{
Assert.That(MimeTypeMap.GetExtension("image/bread", false), Is.EqualTo(DefaultExtension));
Assert.That(MimeTypeMap.GetExtensions("image/bread"), Is.Empty);
}
[Test]
public void GetExtension_MissingMimeTypeShouldCauseArgumentException()
{
Assert.Throws<ArgumentException>(() => MimeTypeMap.GetExtension("image/bread"));
}
{% for gkv in mime_types -%}
{% for kv in gkv.Value -%}
[TestCase(MimeTypeMap.{{ gkv.Key | sharpify }}.{{ kv.Key | sharpify }})]
......@@ -68,11 +52,13 @@ namespace PommaLabs.MimeTypes.UnitTests
// in the README.
var isDeprecatedMimeType = mimeType switch
{
"application/onenote" => true,
"application/pkcs7-mime" => true,
"application/vnd.ms-works" => true,
"application/x-msmediaview" => true,
"application/x-compressed" => true,
"application/x-x509-ca-cert" => true,
"audio/x-pn-realaudio-plugin" => true,
"video/3gpp2" => true,
"x-world/x-vrml" => true,
_ => false,
};
......@@ -81,7 +67,7 @@ namespace PommaLabs.MimeTypes.UnitTests
Assert.Pass($"MediaTypeMap.Core probably returns a wrong extension for {mimeType}");
}
var mediaTypeMapCoreResult = global::MimeTypes.MimeTypeMap.GetExtension(mimeType, false);
var mediaTypeMapCoreResult = global::MimeTypes.MimeTypeMap.GetExtension(mimeType, false).ToLowerInvariant();
if (string.IsNullOrWhiteSpace(mediaTypeMapCoreResult) || mediaTypeMapCoreResult == DefaultExtension)
{
Assert.Pass($"MediaTypeMap.Core probably does not support mime type {mimeType}");
......@@ -93,7 +79,7 @@ namespace PommaLabs.MimeTypes.UnitTests
{% for gkv in mime_types -%}
{% for kv in gkv.Value -%}
[TestCase(MimeTypeMap.{{ gkv.Key | sharpify }}.{{ kv.Key | sharpify }}, ".{{ kv.Value }}")]
[TestCase(MimeTypeMap.{{ gkv.Key | sharpify }}.{{ kv.Key | sharpify }}, ".{{ kv.Value | first }}")]
{% endfor -%}
{% endfor -%}
public void GetExtensionWithoutDot_ExistingMimeTypeShouldBeMatchedToCorrectExtension(string mimeType, string extension)
......@@ -102,37 +88,47 @@ namespace PommaLabs.MimeTypes.UnitTests
Assert.That(MimeTypeMap.GetExtensionsWithoutDot(mimeType), Contains.Item(extension.Substring(1)));
}
[Test]
public void GetExtensionWithoutDot_MissingMimeTypeShouldBeMatchedToBinExtensionWhenThrowExceptionIsFalse()
{
Assert.That(MimeTypeMap.GetExtensionWithoutDot("image/bread", false), Is.EqualTo(DefaultExtension.Substring(1)));
Assert.That(MimeTypeMap.GetExtensionsWithoutDot("image/bread"), Is.Empty);
}
[Test]
public void GetExtensionWithoutDot_MissingMimeTypeShouldCauseArgumentException()
{
Assert.Throws<ArgumentException>(() => MimeTypeMap.GetExtensionWithoutDot("image/bread"));
}
{% for kv in extensions -%}
[TestCase(MimeTypeMap.Extensions.{{ kv.Key | sharpify }}, "{{ kv.Value }}")]
[TestCase(MimeTypeMap.Extensions.{{ kv.Key | sharpify }}, "{{ kv.Value | first }}")]
{% endfor -%}
public void GetMimeType_ExistingExtensionShouldBeMatchedToCorrectMimeType(string extension, string mimeType)
{
Assert.That(MimeTypeMap.GetMimeType($"file{extension}"), Is.EqualTo(mimeType));
}
[Test]
public void GetMimeType_MissingExtensionShouldBeMatchedToApplicationOctetStreamMimeTypeWhenThrowExceptionIsFalse()
{% for kv in extensions -%}
[TestCase(MimeTypeMap.Extensions.{{ kv.Key | sharpify }})]
{% endfor -%}
public void GetMimeType_ResultShouldBeConsistentWithMediaTypeMapCore(string extension)
{
Assert.That(MimeTypeMap.GetMimeType("file.bread", false), Is.EqualTo(DefaultMimeType));
}
// As documented in this project README, not all mime types could be mapped
// consistently. When adding new mime types here, please ensure that they are documented
// in the README.
/*var isDeprecatedMimeType = mimeType switch
{
"application/onenote" => true,
"application/pkcs7-mime" => true,
"application/x-compressed" => true,
"application/x-x509-ca-cert" => true,
"audio/x-pn-realaudio-plugin" => true,
"video/3gpp2" => true,
"x-world/x-vrml" => true,
_ => false,
};
[Test]
public void GetMimeType_MissingExtensionShouldCauseArgumentException()
{
Assert.Throws<ArgumentException>(() => MimeTypeMap.GetMimeType("file.bread"));
if (isDeprecatedMimeType)
{
Assert.Pass($"MediaTypeMap.Core probably returns a wrong extension for {mimeType}");
}*/
var mediaTypeMapCoreResult = global::MimeTypes.MimeTypeMap.GetMimeType(extension).ToLowerInvariant();
if (string.IsNullOrWhiteSpace(mediaTypeMapCoreResult) || mediaTypeMapCoreResult == DefaultMimeType)
{
Assert.Pass($"MediaTypeMap.Core probably does not support extension {extension}");
}
var mimeTypesResult = MimeTypeMap.GetMimeType(extension);
Assert.That(mimeTypesResult, Is.EqualTo(mediaTypeMapCoreResult));
}
}
}
......@@ -9,6 +9,7 @@
<ItemGroup>
<PackageReference Include="DotLiquid" Version="2.0.325" />
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
</ItemGroup>
<ItemGroup>
......
......@@ -22,13 +22,14 @@
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text.RegularExpressions;
using DotLiquid;
using StringMap = System.Collections.Generic.SortedList<string, string>;
using StringMultiMap = System.Collections.Generic.SortedList<string, System.Collections.Generic.SortedList<string, string>>;
using Newtonsoft.Json;
using StringMultiMap = System.Collections.Generic.SortedList<string, System.Collections.Generic.List<string>>;
using StringMultiMap2 = System.Collections.Generic.SortedList<string, System.Collections.Generic.SortedList<string, System.Collections.Generic.List<string>>>;
namespace PommaLabs.MimeTypes.Generator
{
......@@ -37,7 +38,7 @@ namespace PommaLabs.MimeTypes.Generator
/// </summary>
public static class Program
{
private const string ApacheMimeTypesUrl = "https://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types";
private static readonly Uri s_sourceDataUri = new Uri("https://raw.githubusercontent.com/mime-types/mime-types-data/master/data/mime-types.json");
/// <summary>
/// Entry point.
......@@ -72,20 +73,20 @@ namespace PommaLabs.MimeTypes.Generator
}
}
private static string GenerateMimeTypeMap(StringMap extensions, StringMultiMap mimeTypes)
private static string GenerateMimeTypeMap(StringMultiMap extensions, StringMultiMap2 mimeTypes)
{
var templateContents = File.ReadAllText("MimeTypeMap.liquid");
var template = Template.Parse(templateContents);
return template.Render(Hash.FromAnonymousObject(new
{
source_url = ApacheMimeTypesUrl,
source_url = s_sourceDataUri.AbsoluteUri,
extensions,
mime_types = mimeTypes,
}));
}
private static string GenerateMimeTypeMapTests(StringMap extensions, StringMultiMap mimeTypes)
private static string GenerateMimeTypeMapTests(StringMultiMap extensions, StringMultiMap2 mimeTypes)
{
var templateContents = File.ReadAllText("MimeTypeMapTests.liquid");
var template = Template.Parse(templateContents);
......@@ -97,53 +98,53 @@ namespace PommaLabs.MimeTypes.Generator
}));
}
private static (StringMap extensions, StringMultiMap mimeTypes) GetExtensionsAndMimeTypes()
private static (StringMultiMap extensions, StringMultiMap2 mimeTypes) GetExtensionsAndMimeTypes()
{
using var webClient = new WebClient();
var response = webClient.DownloadString(new Uri(ApacheMimeTypesUrl));
var response = webClient.DownloadString(s_sourceDataUri);
var extensions = new StringMap(StringComparer.OrdinalIgnoreCase);
var mimeTypes = new StringMultiMap(StringComparer.OrdinalIgnoreCase);
var extensions = new StringMultiMap(StringComparer.OrdinalIgnoreCase);
var mimeTypes = new StringMultiMap2(StringComparer.OrdinalIgnoreCase);
using var reader = new StringReader(response);
var mimeTypeData = JsonConvert.DeserializeObject<MimeTypeDatum[]>(reader.ReadToEnd());
string line;
while ((line = reader.ReadLine()) != null)
foreach (var mimeTypeDatum in mimeTypeData.OrderBy(x => x.ContentType))
{
if (line.Trim().StartsWith("#", StringComparison.OrdinalIgnoreCase))
if (mimeTypeDatum.Extensions == null || mimeTypeDatum.Extensions.Length == 0)
{
continue;
}
var parts = Regex.Split(line, @"\s+");
if (parts.Length < 2)
{
continue;
}
var mimeType = mimeTypeDatum.ContentType.ToLowerInvariant();
var extensionList = mimeTypeDatum.Extensions.Select(x => x.ToLowerInvariant());
var mimeType = parts[0].ToLowerInvariant();
var extensionList = parts[1..].Select(x => x.ToLowerInvariant());
var splitMimeType = mimeType.Split('/');
var type = splitMimeType[0];
var subtype = splitMimeType[1];
var first = true;
foreach (var extension in extensionList)
{
if (!extensions.ContainsKey(extension))
if (!extensions.TryGetValue(extension, out var extensionMimeTypes))
{
extensions.Add(extension, mimeType);
extensions[extension] = extensionMimeTypes = new List<string>();
}
if (first)
if (!extensionMimeTypes.Contains(mimeType, StringComparer.OrdinalIgnoreCase))
{
var splitMimeType = mimeType.Split('/');
var type = splitMimeType[0];
var subtype = splitMimeType[1];
if (!mimeTypes.TryGetValue(type, out var subtypes))
{
mimeTypes[type] = subtypes = new StringMap(StringComparer.OrdinalIgnoreCase);
}
extensionMimeTypes.Add(mimeType);
}
subtypes.Add(subtype, extension);
first = false;
if (!mimeTypes.TryGetValue(type, out var subtypes))
{
mimeTypes[type] = subtypes = new StringMultiMap(StringComparer.OrdinalIgnoreCase);
}
if (!subtypes.TryGetValue(subtype, out var subtypeExtensions))
{
subtypes[subtype] = subtypeExtensions = new List<string>();
}
if (!subtypeExtensions.Contains(extension))
{
subtypeExtensions.Add(extension);
}
}
}
......@@ -151,21 +152,52 @@ namespace PommaLabs.MimeTypes.Generator
// Following overrides ensures best compatibility with MediaTypeMap.Core results.
// Complete compatibility cannot be achieved because some results of that library were
// simply wrong and they are documented in this project README.
extensions["3gp2"] = "video/3gpp2";
extensions["one"] = "application/onenote";
mimeTypes["application"]["onenote"] = "one";
mimeTypes["application"]["postscript"] = "eps";
mimeTypes["application"]["x-msdownload"] = "dll";
mimeTypes["application"]["x-texinfo"] = "texi";
mimeTypes["audio"]["basic"] = "snd";
mimeTypes["audio"]["mpeg"] = "mp3";
mimeTypes["audio"]["x-pn-realaudio"] = "ra";
mimeTypes["image"]["jpeg"] = "jpg";
mimeTypes["video"]["3gpp2"] = "3gp2";
mimeTypes["video"]["mpeg"] = "mpg";
mimeTypes["video"]["quicktime"] = "mov";
Swap(mimeTypes["application"]["vnd.ms-works"], "wcm", "wks");
//mimeTypes["application"]["x-director"] = "dir";
//mimeTypes["application"]["x-latex"] = "latex";
//mimeTypes["application"]["x-msaccess"] = "mdb";
//mimeTypes["application"]["x-msdownload"] = "dll";
//mimeTypes["application"]["x-msmetafile"] = "wmf";
//mimeTypes["application"]["x-texinfo"] = "texi";
//mimeTypes["application"]["x-troff"] = "roff";
//mimeTypes["application"]["xhtml+xml"] = "xhtml";
//mimeTypes["audio"]["basic"] = "snd";
//mimeTypes["audio"]["mpeg"] = "mp3";
//mimeTypes["image"]["jpeg"] = "jpg";
//mimeTypes["video"]["mpeg"] = "mpg";
//mimeTypes["video"]["ogg"] = "ogv";
//mimeTypes["video"]["quicktime"] = "mov";
//mimeTypes["video"]["x-matroska"] = "mkv";
return (extensions, mimeTypes);
}
private static void Swap(List<string> list, string s1, string s2)
{
var i1 = list.IndexOf(s1);
if (i1 != 0)
{
throw new Exception($"{s1} is not the first item of list [{string.Join(", ", list)}]");
}
var i2 = list.IndexOf(s2);
if (i2 < 0)
{
throw new Exception($"{s2} was not found in list [{string.Join(", ", list)}]");
}
list[i1] = s2;
list[i2] = s1;
}
private struct MimeTypeDatum
{
[JsonProperty("content-type")]
public string ContentType { get; set; }
[JsonProperty("extensions")]
public string[] Extensions { get; set; }
}
}
}
{
"profiles": {
"PommaLabs.MimeTypes.Generator": {
"commandName": "Project",
"commandLineArgs": "src"
}
}
}
\ No newline at end of file
This diff is collapsed.
This diff is collapsed.
This source diff could not be displayed because it is too large. You can view the blob instead.
......@@ -6,7 +6,7 @@ $Test = dotnet run -- test | Out-String
$UTF8WithBom = New-Object System.Text.UTF8Encoding $True
[System.Environment]::CurrentDirectory = (Get-Location).Path
[System.IO.File]::WriteAllLines("..\..\src\PommaLabs.MimeTypes\MimeTypeMap.cs", $Src, $UTF8WithBom)
[System.IO.File]::WriteAllLines("..\..\test\PommaLabs.MimeTypes.UnitTests\MimeTypeMapTests.cs", $Test, $UTF8WithBom)
[System.IO.File]::WriteAllLines("..\..\src\PommaLabs.MimeTypes\MimeTypeMap.Generated.cs", $Src, $UTF8WithBom)
[System.IO.File]::WriteAllLines("..\..\test\PommaLabs.MimeTypes.UnitTests\MimeTypeMapTests.Generated.cs", $Test, $UTF8WithBom)
cd ..\..
\ No newline at end of file