Commit 387b2781 authored by Grzegorz Bizon's avatar Grzegorz Bizon

Change format of artifacts metadata from text to binary 0.0.1

This changes the format of metadata to handle paths, that may contain
whitespace characters, new line characters and non-UTF-8 characters.

Now those paths along with metadata in JSON format are stored as
length-prefixed strings (uint32 prefix).

Metadata file has a custom format:

1.   First string field is metadata version field (string)
2.   Second string field is metadata errors field (JSON strong)
3.   All subsequent fields is pair of path (string) and path metadata
     in JSON format.

Path's metadata contains all fields that where possible to extract from
ZIP archive like date of modification, CRC, compressed size,
uncompressed size and comment.
parent 09c82c6f
......@@ -4,7 +4,7 @@
%span.str-truncated
= file.name
%td
= number_to_human_size(file.metadata[:uncompressed_size], precision: 2)
= number_to_human_size(file.metadata[:size], precision: 2)
%td
= link_to '', class: 'btn btn-xs btn-default' do
= icon('download')
......@@ -17,18 +17,33 @@ module Gitlab
File.exists?(@file)
end
def full_version
gzip do|gz|
read_string(gz) do |size|
raise StandardError, 'Artifacts metadata file empty!' unless size
end
end
end
def version
full_version.match(/\w+ (\d+\.\d+\.\d+)/).captures.first
end
def errors
gzip do|gz|
read_string(gz) # version
JSON.parse(read_string(gz))
end
end
def match!
raise StandardError, 'Metadata file not found !' unless exists?
paths, metadata = [], []
each do |line|
next unless line =~ %r{^#{Regexp.escape(@path)}[^/\s]*/?\s}
path, meta = line.split(' ')
paths.push(path)
metadata.push(meta)
gzip do |gz|
read_string(gz) # version field
read_string(gz) # errors field
iterate_entries(gz)
end
[paths, metadata.map { |meta| JSON.parse(meta, symbolize_names: true) }]
end
def to_string_path
......@@ -38,11 +53,44 @@ module Gitlab
private
def each
def iterate_entries(gz)
paths, metadata = [], []
until gz.eof? do
begin
path = read_string(gz)
meta = read_string(gz)
next unless path =~ %r{^#{Regexp.escape(@path)}[^/\s]*/?$}
paths.push(path)
metadata.push(JSON.parse(meta, symbolize_names: true))
rescue JSON::ParserError
next
end
end
[paths, metadata]
end
def read_string_size(gz)
binary = gz.read(4)
binary.unpack('L>')[0] if binary
end
def read_string(gz)
string_size = read_string_size(gz)
yield string_size if block_given?
return false unless string_size
gz.read(string_size).chomp
end
def gzip
open do |file|
gzip = Zlib::GzipReader.new(file)
gzip.each_line { |line| yield line }
result = yield gzip
gzip.close
result
end
end
......
......@@ -59,6 +59,21 @@ describe Gitlab::Ci::Build::Artifacts::Metadata do
subject { metadata('').to_string_path }
it { is_expected.to be_an_instance_of(Gitlab::StringPath) }
end
describe '#full_version' do
subject { metadata('').full_version }
it { is_expected.to eq 'GitLab Build Artifacts Metadata 0.0.1' }
end
describe '#version' do
subject { metadata('').version }
it { is_expected.to eq '0.0.1' }
end
describe '#errors' do
subject { metadata('').errors }
it { is_expected.to eq({}) }
end
end
context 'metadata file does not exist' do
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment