|
1 | 1 | import asyncio |
2 | | -import base64 |
3 | 2 | import gzip |
4 | | -import hashlib |
5 | 3 | import itertools |
6 | 4 | import json |
7 | 5 | import os |
8 | 6 | import tempfile |
9 | 7 |
|
10 | | -import aiohttp |
11 | 8 | import arrow |
12 | 9 | import mock |
13 | 10 | import pytest |
|
21 | 18 | download_artifacts, |
22 | 19 | get_and_check_single_upstream_artifact_full_path, |
23 | 20 | get_artifact_url, |
24 | | - get_content_md5, |
25 | 21 | get_expiration_arrow, |
26 | 22 | get_optional_artifacts_per_task_id, |
27 | 23 | get_single_upstream_artifact_full_path, |
|
31 | 27 | ) |
32 | 28 | from scriptworker.exceptions import ScriptWorkerRetryException, ScriptWorkerTaskException |
33 | 29 |
|
34 | | -from . import FakeResponse, touch |
| 30 | +from . import touch |
35 | 31 |
|
36 | 32 |
|
37 | 33 | @pytest.fixture(scope="function") |
@@ -166,81 +162,6 @@ async def test_create_artifact_retry(context, fake_session_500, successful_queue |
166 | 162 | await create_artifact(context, path, "public/env/one.log", content_type="text/plain", content_encoding=None, expires=expires) |
167 | 163 |
|
168 | 164 |
|
169 | | -def _write(path, contents=b"hello world"): |
170 | | - with open(path, "wb") as fh: |
171 | | - fh.write(contents) |
172 | | - return contents |
173 | | - |
174 | | - |
175 | | -async def _capture_put(context, fake_session, successful_queue, path, target_path="public/env/one.txt", content_type="text/plain", content_encoding=None): |
176 | | - captured = {} |
177 | | - original = fake_session._request |
178 | | - |
179 | | - async def capture(method, url, *args, **kwargs): |
180 | | - captured.update(kwargs) |
181 | | - return await original(method, url, *args, **kwargs) |
182 | | - |
183 | | - fake_session._request = capture |
184 | | - context.session = fake_session |
185 | | - context.temp_queue = successful_queue |
186 | | - await create_artifact(context, path, target_path, content_type=content_type, content_encoding=content_encoding, expires=arrow.utcnow().isoformat()) |
187 | | - return captured |
188 | | - |
189 | | - |
190 | | -@pytest.mark.asyncio |
191 | | -async def test_create_artifact_sends_content_md5(context, fake_session, successful_queue): |
192 | | - path = os.path.join(context.config["artifact_dir"], "one.txt") |
193 | | - contents = _write(path) |
194 | | - |
195 | | - captured = await _capture_put(context, fake_session, successful_queue, path) |
196 | | - |
197 | | - expected = base64.b64encode(hashlib.md5(contents).digest()).decode("ascii") |
198 | | - assert captured["headers"][aiohttp.hdrs.CONTENT_MD5] == expected |
199 | | - |
200 | | - |
201 | | -@pytest.mark.asyncio |
202 | | -async def test_create_artifact_content_md5_covers_compressed_bytes(context, fake_session, successful_queue): |
203 | | - """The digest has to cover what goes on the wire, which for a gzipped artifact is the compressed file.""" |
204 | | - path = os.path.join(context.config["artifact_dir"], "one.log") |
205 | | - original_contents = _write(path, b"12:00:00 Foo bar") |
206 | | - content_type, content_encoding = compress_artifact_if_supported(path) |
207 | | - assert content_encoding == "gzip" |
208 | | - |
209 | | - captured = await _capture_put( |
210 | | - context, fake_session, successful_queue, path, target_path="public/logs/one.log", content_type=content_type, content_encoding=content_encoding |
211 | | - ) |
212 | | - |
213 | | - with open(path, "rb") as fh: |
214 | | - on_disk = fh.read() |
215 | | - assert on_disk != original_contents |
216 | | - assert captured["headers"][aiohttp.hdrs.CONTENT_MD5] == base64.b64encode(hashlib.md5(on_disk).digest()).decode("ascii") |
217 | | - assert captured["headers"][aiohttp.hdrs.CONTENT_ENCODING] == "gzip" |
218 | | - |
219 | | - |
220 | | -@pytest.mark.asyncio |
221 | | -async def test_create_artifact_bad_digest_retries(context, fake_session, successful_queue, caplog): |
222 | | - path = os.path.join(context.config["artifact_dir"], "one.txt") |
223 | | - _write(path) |
224 | | - |
225 | | - async def bad_digest(method, url, *args, **kwargs): |
226 | | - return FakeResponse(method, url, status=400, payload="<Error><Code>BadDigest</Code></Error>") |
227 | | - |
228 | | - fake_session._request = bad_digest |
229 | | - context.session = fake_session |
230 | | - context.temp_queue = successful_queue |
231 | | - |
232 | | - with pytest.raises(ScriptWorkerRetryException): |
233 | | - await create_artifact(context, path, "public/env/one.txt", content_type="text/plain", content_encoding=None, expires=arrow.utcnow().isoformat()) |
234 | | - |
235 | | - assert "was corrupted in transit" in caplog.text |
236 | | - |
237 | | - |
238 | | -def test_get_content_md5(tmpdir): |
239 | | - path = os.path.join(tmpdir, "one.txt") |
240 | | - contents = _write(path, b"some artifact contents") |
241 | | - assert get_content_md5(path) == base64.b64encode(hashlib.md5(contents).digest()).decode("ascii") |
242 | | - |
243 | | - |
244 | 165 | @pytest.mark.asyncio |
245 | 166 | async def test_create_link_artifact(context, successful_queue): |
246 | 167 | expires = arrow.utcnow().isoformat() |
@@ -270,13 +191,9 @@ async def test_create_link_artifact(context, successful_queue): |
270 | 191 |
|
271 | 192 |
|
272 | 193 | def test_craft_artifact_put_headers(): |
273 | | - assert _craft_artifact_put_headers("text/plain", "deadbeef==") == {"Content-Type": "text/plain", "Content-MD5": "deadbeef=="} |
274 | | - assert _craft_artifact_put_headers("text/plain", "deadbeef==", encoding=None) == {"Content-Type": "text/plain", "Content-MD5": "deadbeef=="} |
275 | | - assert _craft_artifact_put_headers("text/plain", "deadbeef==", "gzip") == { |
276 | | - "Content-Type": "text/plain", |
277 | | - "Content-Encoding": "gzip", |
278 | | - "Content-MD5": "deadbeef==", |
279 | | - } |
| 194 | + assert _craft_artifact_put_headers("text/plain") == {"Content-Type": "text/plain"} |
| 195 | + assert _craft_artifact_put_headers("text/plain", encoding=None) == {"Content-Type": "text/plain"} |
| 196 | + assert _craft_artifact_put_headers("text/plain", "gzip") == {"Content-Type": "text/plain", "Content-Encoding": "gzip"} |
280 | 197 |
|
281 | 198 |
|
282 | 199 | # get_artifact_url {{{1 |
|
0 commit comments