Skip to content

Commit 985a816

Browse files
authored
Merge branch 'main' into CM-71880-guardrails-verdict-plane
2 parents e244196 + b42761d commit 985a816

9 files changed

Lines changed: 448 additions & 31 deletions

File tree

‎cycode/cli/apps/scan/code_scanner.py‎

Lines changed: 76 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import os
22
import time
3+
import zipfile
34
from platform import platform
45
from typing import TYPE_CHECKING, Callable, Optional
56

@@ -23,6 +24,7 @@
2324
from cycode.cli.files_collector.sca.sca_file_collector import add_sca_dependencies_tree_documents_if_needed
2425
from cycode.cli.files_collector.zip_documents import zip_documents
2526
from cycode.cli.models import CliError, Document, LocalScanResult
27+
from cycode.cli.utils.host_info import is_64bit
2628
from cycode.cli.utils.path_utils import get_absolute_path, get_path_by_os
2729
from cycode.cli.utils.progress_bar import ScanProgressBarSection
2830
from cycode.cli.utils.scan_batch import run_parallel_batched_scan
@@ -145,6 +147,7 @@ def _get_scan_documents_thread_func(
145147
is_git_diff: bool,
146148
is_commit_range: bool,
147149
scan_parameters: dict,
150+
prezipped: Optional['InMemoryZip'] = None,
148151
) -> Callable[[list[Document]], tuple[str, CliError, LocalScanResult]]:
149152
cycode_client = ctx.obj['client']
150153
scan_type = ctx.obj['scan_type']
@@ -164,9 +167,14 @@ def _scan_batch_thread_func(batch: list[Document]) -> tuple[str, CliError, Local
164167

165168
should_use_sync_flow = _should_use_sync_flow(command_scan_type, scan_type, sync_option)
166169

170+
# the single ZIP flow already built the archive to check that it fits; don't build it twice
171+
zipped_documents = prezipped
172+
167173
try:
168-
logger.debug('Preparing local files, %s', {'batch_files_count': len(batch)})
169-
zipped_documents = zip_documents(scan_type, batch)
174+
if zipped_documents is None:
175+
logger.debug('Preparing local files, %s', {'batch_files_count': len(batch)})
176+
zipped_documents = zip_documents(scan_type, batch)
177+
170178
zip_file_size = zipped_documents.size
171179
scan_result = _perform_scan(
172180
cycode_client,
@@ -189,6 +197,9 @@ def _scan_batch_thread_func(batch: list[Document]) -> tuple[str, CliError, Local
189197
except Exception as e:
190198
error = handle_scan_exception(ctx, e, return_exception=True)
191199
error_message = str(e)
200+
finally:
201+
if zipped_documents is not None:
202+
zipped_documents.cleanup()
192203

193204
if local_scan_result:
194205
detections_count = local_scan_result.detections_count
@@ -225,34 +236,77 @@ def _scan_batch_thread_func(batch: list[Document]) -> tuple[str, CliError, Local
225236
return _scan_batch_thread_func
226237

227238

239+
def _log_selected_upload_mode(mode: str, reason: str, documents_count: int) -> None:
240+
logger.debug(
241+
'Selected upload mode, %s',
242+
{
243+
'mode': mode,
244+
'reason': reason,
245+
'documents_count': documents_count,
246+
'max_files_count': consts.ZIP_MAX_FILES_COUNT,
247+
'zip64_enabled': is_64bit(),
248+
},
249+
)
250+
251+
252+
def _exceeds_non_zip64_files_count(documents_to_scan: list[Document]) -> bool:
253+
"""Whether a single ZIP can't hold all the documents because ZIP64 is unavailable.
254+
255+
Without ZIP64 (32-bit interpreter) the archive is capped at 65,535 entries.
256+
"""
257+
return not is_64bit() and len(documents_to_scan) > consts.ZIP_MAX_FILES_COUNT
258+
259+
228260
def _run_presigned_upload_scan(
229-
scan_batch_thread_func: Callable,
230-
scan_type: str,
261+
ctx: typer.Context,
262+
is_git_diff: bool,
263+
is_commit_range: bool,
264+
scan_parameters: dict,
231265
documents_to_scan: list[Document],
232266
progress_bar: 'BaseProgressBar',
233267
printer: 'ConsolePrinter',
234268
) -> tuple:
235-
try:
236-
# Try to zip all documents as a single batch; ZipTooLargeError raised if it exceeds the scan type's limit
237-
zip_documents(scan_type, documents_to_scan)
238-
# It fits: skip batching and upload everything as one ZIP
269+
scan_type = ctx.obj['scan_type']
270+
documents_count = len(documents_to_scan)
271+
272+
def run_batched() -> tuple:
239273
return run_parallel_batched_scan(
240-
scan_batch_thread_func,
274+
_get_scan_documents_thread_func(ctx, is_git_diff, is_commit_range, scan_parameters),
241275
scan_type,
242276
documents_to_scan,
243277
progress_bar=progress_bar,
244-
skip_batching=True,
245278
)
246-
except custom_exceptions.ZipTooLargeError:
279+
280+
if _exceeds_non_zip64_files_count(documents_to_scan):
281+
# Don't waste time zipping documents we already know won't fit into a single ZIP
282+
_log_selected_upload_mode('batched', 'files_count_exceeds_non_zip64_limit', documents_count)
283+
return run_batched()
284+
285+
zipped_documents = None
286+
try:
287+
# Try to zip all documents as a single batch; ZipTooLargeError raised if it exceeds the scan type's limit
288+
zipped_documents = zip_documents(scan_type, documents_to_scan)
289+
except (custom_exceptions.ZipTooLargeError, zipfile.LargeZipFile):
290+
# LargeZipFile is a safety net: the files count pre-check above should have caught it already
291+
_log_selected_upload_mode('batched', 'zip_too_large', documents_count)
292+
if zipped_documents is not None:
293+
zipped_documents.cleanup()
294+
247295
printer.print_warning(
248296
'The scan is too large to upload as a single file. This may result in corrupted scan results.'
249297
)
250-
return run_parallel_batched_scan(
251-
scan_batch_thread_func,
252-
scan_type,
253-
documents_to_scan,
254-
progress_bar=progress_bar,
255-
)
298+
return run_batched()
299+
300+
# It fits: skip batching and upload everything as one ZIP. The archive we just built is the one
301+
# that gets uploaded, so the scan doesn't pay for compressing every document twice
302+
_log_selected_upload_mode('single_zip', 'fits_single_zip', documents_count)
303+
return run_parallel_batched_scan(
304+
_get_scan_documents_thread_func(ctx, is_git_diff, is_commit_range, scan_parameters, zipped_documents),
305+
scan_type,
306+
documents_to_scan,
307+
progress_bar=progress_bar,
308+
skip_batching=True,
309+
)
256310

257311

258312
def scan_documents(
@@ -277,18 +331,19 @@ def scan_documents(
277331
)
278332
return
279333

280-
scan_batch_thread_func = _get_scan_documents_thread_func(ctx, is_git_diff, is_commit_range, scan_parameters)
281-
282334
# Presigned single-file upload is async-only; a --sync scan must stay on the batched inline path
283335
# so it never builds one oversized zip to POST synchronously.
284336
should_use_sync_flow = _should_use_sync_flow(ctx.info_name, scan_type, ctx.obj['sync'])
285337
if should_use_presigned_upload(scan_type) and not should_use_sync_flow:
286338
errors, local_scan_results = _run_presigned_upload_scan(
287-
scan_batch_thread_func, scan_type, documents_to_scan, progress_bar, printer
339+
ctx, is_git_diff, is_commit_range, scan_parameters, documents_to_scan, progress_bar, printer
288340
)
289341
else:
290342
errors, local_scan_results = run_parallel_batched_scan(
291-
scan_batch_thread_func, scan_type, documents_to_scan, progress_bar=progress_bar
343+
_get_scan_documents_thread_func(ctx, is_git_diff, is_commit_range, scan_parameters),
344+
scan_type,
345+
documents_to_scan,
346+
progress_bar=progress_bar,
292347
)
293348

294349
try_set_aggregation_report_url_if_needed(ctx, scan_parameters, ctx.obj['client'], scan_type)

‎cycode/cli/apps/scan/commit_range_scanner.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,9 @@ def _scan_commit_range_documents(
214214

215215
zip_file_size = from_commit_zipped_documents.size + to_commit_zipped_documents.size
216216

217+
from_commit_zipped_documents.cleanup()
218+
to_commit_zipped_documents.cleanup()
219+
217220
detections_count = relevant_detections_count = 0
218221
if local_scan_result:
219222
detections_count = local_scan_result.detections_count

‎cycode/cli/consts.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,12 @@
227227
PRESIGNED_LINK_UPLOADED_ZIP_MAX_SIZE_LIMIT_IN_BYTES = 5 * 1024 * 1024 * 1024 # 5 GB (S3 presigned POST limit)
228228
PRESIGNED_UPLOAD_SCAN_TYPES = {SAST_SCAN_TYPE, SECRET_SCAN_TYPE}
229229

230+
# the non-ZIP64 central directory stores the entry count in 16 bits; ZIP64 (64-bit interpreters) lifts it
231+
ZIP_MAX_FILES_COUNT = 65_535
232+
233+
# the ZIP is built in memory up to this size, and spilled to a temp file beyond it
234+
ZIP_SPOOL_MAX_SIZE_IN_BYTES = 64 * 1024 * 1024
235+
230236
DEFAULT_ZIP_MAX_SIZE_LIMIT_IN_BYTES = 20 * 1024 * 1024
231237
ZIP_MAX_SIZE_LIMIT_IN_BYTES = {
232238
SCA_SCAN_TYPE: 200 * 1024 * 1024,

‎cycode/cli/exceptions/handle_scan_errors.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import zipfile
12
from typing import Optional
23

34
import typer
@@ -26,6 +27,13 @@ def handle_scan_exception(ctx: typer.Context, err: Exception, *, return_exceptio
2627
'Please try ignoring irrelevant paths using the `cycode ignore --by-path` command '
2728
'and execute the scan again',
2829
),
30+
zipfile.LargeZipFile: CliError(
31+
soft_fail=True,
32+
code='zip_too_large_error',
33+
message='The path you attempted to scan contains too many files to pack into a single archive. '
34+
'Scanning such paths requires a 64-bit Python interpreter. '
35+
'Please try ignoring irrelevant paths using a .cycodeignore file and execute the scan again',
36+
),
2937
custom_exceptions.FileCollectionError: CliError(
3038
soft_fail=False,
3139
code='file_collection_error',

‎cycode/cli/files_collector/models/in_memory_zip.py‎

Lines changed: 74 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,48 @@
1+
import shutil
2+
import tempfile
13
from collections import defaultdict
2-
from io import BytesIO
4+
from os import SEEK_END
35
from pathlib import Path
4-
from sys import getsizeof
5-
from typing import Optional
6+
from typing import IO, Optional
67
from zipfile import ZIP_DEFLATED, ZipFile
78

9+
from cycode.cli import consts
810
from cycode.cli.user_settings.configuration_manager import ConfigurationManager
11+
from cycode.cli.utils.host_info import is_64bit
912
from cycode.cli.utils.path_utils import concat_unique_id
13+
from cycode.logger import get_logger
14+
15+
logger = get_logger('ZIP')
16+
17+
_SPOOL_DIRECTORY_NAME = 'tmp'
18+
19+
20+
def _get_spool_directory(configuration_manager: ConfigurationManager) -> Optional[str]:
21+
"""Directory to spill big ZIPs into. None falls back to the system temp directory."""
22+
try:
23+
directory = Path(configuration_manager.global_config_file_manager.get_config_directory_path())
24+
spool_directory = directory / _SPOOL_DIRECTORY_NAME
25+
spool_directory.mkdir(parents=True, exist_ok=True)
26+
return str(spool_directory)
27+
except OSError as e:
28+
logger.debug('Failed to create the spool directory; falling back to the system one', exc_info=e)
29+
return None
1030

1131

1232
class InMemoryZip:
1333
def __init__(self) -> None:
1434
self.configuration_manager = ConfigurationManager()
1535

16-
self.in_memory_zip = BytesIO()
17-
self.zip = ZipFile(self.in_memory_zip, mode='a', compression=ZIP_DEFLATED, allowZip64=False)
36+
self._spool_max_size = consts.ZIP_SPOOL_MAX_SIZE_IN_BYTES
37+
self._buffer = tempfile.SpooledTemporaryFile( # noqa: SIM115 # closed by cleanup(), lives past close()
38+
max_size=self._spool_max_size,
39+
dir=_get_spool_directory(self.configuration_manager),
40+
)
41+
42+
# ZIP64 lifts the 65,535 entries and 4 GiB caps of the original ZIP format.
43+
# It requires 64-bit offsets, so we only enable it on a 64-bit interpreter.
44+
self._allow_zip64 = is_64bit()
45+
self.zip = ZipFile(self._buffer, mode='a', compression=ZIP_DEFLATED, allowZip64=self._allow_zip64)
1846

1947
self._files_count = 0
2048
self._extension_statistics = defaultdict(int)
@@ -35,17 +63,54 @@ def append(self, filename: str, unique_id: Optional[str], content: str) -> None:
3563
def close(self) -> None:
3664
self.zip.close()
3765

66+
def cleanup(self) -> None:
67+
"""Release the buffer, deleting the spilled temp file if there is one."""
68+
self._buffer.close()
69+
70+
def __enter__(self) -> 'InMemoryZip': # noqa: PYI034 # typing.Self needs Python 3.11
71+
return self
72+
73+
def __exit__(self, *_: object) -> None:
74+
self.cleanup()
75+
76+
def stream(self) -> IO[bytes]:
77+
"""The whole archive as a file object, rewound. Doesn't copy it into memory.
78+
79+
Note: before Python 3.11 SpooledTemporaryFile isn't a real IOBase, so the returned object
80+
has no seekable()/readable()/writable(). read/seek/tell work on every supported version.
81+
"""
82+
self._buffer.seek(0)
83+
return self._buffer
84+
3885
def read(self) -> bytes:
39-
self.in_memory_zip.seek(0)
40-
return self.in_memory_zip.read()
86+
self._buffer.seek(0)
87+
return self._buffer.read()
4188

4289
def write_on_disk(self, path: 'Path') -> None:
4390
with open(path, 'wb') as f:
44-
f.write(self.read())
91+
shutil.copyfileobj(self.stream(), f)
4592

4693
@property
4794
def size(self) -> int:
48-
return getsizeof(self.in_memory_zip)
95+
position = self._buffer.tell()
96+
try:
97+
self._buffer.seek(0, SEEK_END)
98+
return self._buffer.tell()
99+
finally:
100+
self._buffer.seek(position)
101+
102+
@property
103+
def is_rolled_over(self) -> bool:
104+
"""Whether the archive outgrew the threshold and moved from memory to the disk.
105+
106+
SpooledTemporaryFile spills on the write that crosses max_size, and the archive only grows,
107+
so the size says it without reaching into the private _rolled flag.
108+
"""
109+
return self.size > self._spool_max_size
110+
111+
@property
112+
def allow_zip64(self) -> bool:
113+
return self._allow_zip64
49114

50115
@property
51116
def files_count(self) -> int:

‎cycode/cli/utils/host_info.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,11 @@ def _read_text_file(path: str) -> Optional[str]:
4949
return None
5050

5151

52+
def is_64bit() -> bool:
53+
"""Whether the running Python interpreter is 64-bit (not the OS)."""
54+
return sys.maxsize > 2**32
55+
56+
5257
def get_hostname() -> Optional[str]:
5358
try:
5459
return socket.gethostname() or None

0 commit comments

Comments
 (0)