From fddaca8ee2176cd64394e623b6df1ddb3432e9ab Mon Sep 17 00:00:00 2001 From: hongzhouzi <36416216+hongzhouzi@users.noreply.github.com> Date: Wed, 21 Dec 2022 10:37:53 +0800 Subject: [PATCH] Upgrade dependent version: github.com/containerd/containerd (#5426) Upgrade dependent version: github.com/containerd/containerd v1.4.13 -> v1.5.16 Signed-off-by: hongzhouzi Signed-off-by: hongzhouzi --- .../github.com/Microsoft/hcsshim/LICENSE | 24 - .../github.com/klauspost/compress/LICENSE | 32 + .../vendor/github.com/moby/locker/LICENSE | 194 ++++ go.mod | 42 +- go.sum | 54 +- staging/src/kubesphere.io/api/go.mod | 3 +- staging/src/kubesphere.io/api/go.sum | 4 +- staging/src/kubesphere.io/utils/go.mod | 16 +- staging/src/kubesphere.io/utils/go.sum | 15 +- .../github.com/Microsoft/hcsshim/.gitignore | 1 - .../Microsoft/hcsshim/.gometalinter.json | 17 - vendor/github.com/Microsoft/hcsshim/LICENSE | 21 - vendor/github.com/Microsoft/hcsshim/README.md | 41 - .../github.com/Microsoft/hcsshim/appveyor.yml | 29 - .../github.com/Microsoft/hcsshim/container.go | 192 ---- vendor/github.com/Microsoft/hcsshim/errors.go | 257 ------ .../Microsoft/hcsshim/functional_tests.ps1 | 12 - .../github.com/Microsoft/hcsshim/hcsshim.go | 28 - .../Microsoft/hcsshim/hnsendpoint.go | 94 -- .../Microsoft/hcsshim/hnsglobals.go | 16 - .../Microsoft/hcsshim/hnsnetwork.go | 36 - .../github.com/Microsoft/hcsshim/hnspolicy.go | 57 -- .../Microsoft/hcsshim/hnspolicylist.go | 47 - .../Microsoft/hcsshim/hnssupport.go | 13 - .../github.com/Microsoft/hcsshim/interface.go | 114 --- .../hcsshim/internal/guestrequest/types.go | 100 --- .../Microsoft/hcsshim/internal/guid/guid.go | 69 -- .../hcsshim/internal/hcs/callback.go | 104 --- .../Microsoft/hcsshim/internal/hcs/cgo.go | 7 - .../Microsoft/hcsshim/internal/hcs/errors.go | 287 ------ .../Microsoft/hcsshim/internal/hcs/hcs.go | 48 - .../Microsoft/hcsshim/internal/hcs/log.go | 20 - .../Microsoft/hcsshim/internal/hcs/process.go | 459 ---------- .../Microsoft/hcsshim/internal/hcs/system.go | 685 -------------- .../Microsoft/hcsshim/internal/hcs/utils.go | 33 - .../hcsshim/internal/hcs/waithelper.go | 63 -- .../Microsoft/hcsshim/internal/hcs/watcher.go | 41 - .../hcsshim/internal/hcs/zsyscall_windows.go | 533 ----------- .../hcsshim/internal/hcserror/hcserror.go | 47 - .../Microsoft/hcsshim/internal/hns/hns.go | 23 - .../hcsshim/internal/hns/hnsendpoint.go | 262 ------ .../hcsshim/internal/hns/hnsfuncs.go | 42 - .../hcsshim/internal/hns/hnsglobals.go | 28 - .../hcsshim/internal/hns/hnsnetwork.go | 141 --- .../hcsshim/internal/hns/hnspolicy.go | 98 -- .../hcsshim/internal/hns/hnspolicylist.go | 201 ----- .../hcsshim/internal/hns/hnssupport.go | 49 - .../hcsshim/internal/hns/namespace.go | 110 --- .../hcsshim/internal/hns/zsyscall_windows.go | 76 -- .../hcsshim/internal/interop/interop.go | 27 - .../internal/interop/zsyscall_windows.go | 48 - .../hcsshim/internal/logfields/fields.go | 32 - .../hcsshim/internal/longpath/longpath.go | 24 - .../hcsshim/internal/mergemaps/merge.go | 52 -- .../hcsshim/internal/safefile/safeopen.go | 431 --------- .../internal/safefile/zsyscall_windows.go | 79 -- .../hcsshim/internal/schema1/schema1.go | 245 ----- .../hcsshim/internal/schema2/attachment.go | 31 - .../hcsshim/internal/schema2/battery.go | 13 - .../schema2/cache_query_stats_response.go | 19 - .../hcsshim/internal/schema2/chipset.go | 27 - .../hcsshim/internal/schema2/close_handle.go | 15 - .../hcsshim/internal/schema2/com_port.go | 18 - .../internal/schema2/compute_system.go | 27 - .../hcsshim/internal/schema2/configuration.go | 72 -- .../hcsshim/internal/schema2/console_size.go | 17 - .../hcsshim/internal/schema2/container.go | 35 - .../container_credential_guard_state.go | 25 - .../schema2/container_memory_information.go | 26 - .../hcsshim/internal/schema2/device.go | 16 - .../hcsshim/internal/schema2/devices.go | 43 - .../internal/schema2/enhanced_mode_video.go | 15 - .../internal/schema2/flexible_io_device.go | 19 - .../internal/schema2/guest_connection.go | 19 - .../internal/schema2/guest_connection_info.go | 21 - .../internal/schema2/guest_crash_reporting.go | 15 - .../hcsshim/internal/schema2/guest_os.go | 15 - .../hcsshim/internal/schema2/guest_state.go | 22 - .../hcsshim/internal/schema2/hosted_system.go | 17 - .../hcsshim/internal/schema2/hv_socket.go | 17 - .../hcsshim/internal/schema2/hv_socket_2.go | 16 - .../schema2/hv_socket_service_config.go | 22 - .../schema2/hv_socket_system_config.go | 22 - .../hcsshim/internal/schema2/keyboard.go | 13 - .../hcsshim/internal/schema2/layer.go | 22 - .../internal/schema2/linux_kernel_direct.go | 18 - .../internal/schema2/mapped_directory.go | 21 - .../hcsshim/internal/schema2/mapped_pipe.go | 19 - .../hcsshim/internal/schema2/memory.go | 15 - .../hcsshim/internal/schema2/memory_2.go | 25 - .../schema2/memory_information_for_vm.go | 19 - .../hcsshim/internal/schema2/memory_stats.go | 20 - .../schema2/modify_setting_request.go | 20 - .../hcsshim/internal/schema2/mouse.go | 13 - .../internal/schema2/network_adapter.go | 17 - .../hcsshim/internal/schema2/networking.go | 24 - .../internal/schema2/pause_notification.go | 16 - .../hcsshim/internal/schema2/pause_options.go | 18 - .../hcsshim/internal/schema2/plan9.go | 15 - .../hcsshim/internal/schema2/plan9_share.go | 33 - .../internal/schema2/process_details.go | 34 - .../schema2/process_modify_request.go | 20 - .../internal/schema2/process_parameters.go | 47 - .../internal/schema2/process_status.go | 22 - .../hcsshim/internal/schema2/processor.go | 19 - .../hcsshim/internal/schema2/processor_2.go | 21 - .../internal/schema2/processor_stats.go | 20 - .../hcsshim/internal/schema2/properties.go | 47 - .../internal/schema2/property_query.go | 16 - .../schema2/rdp_connection_options.go | 17 - .../internal/schema2/registry_changes.go | 17 - .../hcsshim/internal/schema2/registry_key.go | 19 - .../internal/schema2/registry_value.go | 31 - .../hcsshim/internal/schema2/restore_state.go | 19 - .../hcsshim/internal/schema2/save_options.go | 19 - .../hcsshim/internal/schema2/scsi.go | 16 - .../schema2/shared_memory_configuration.go | 15 - .../internal/schema2/shared_memory_region.go | 23 - .../schema2/shared_memory_region_info.go | 17 - .../internal/schema2/silo_properties.go | 18 - .../hcsshim/internal/schema2/statistics.go | 30 - .../hcsshim/internal/schema2/storage.go | 21 - .../hcsshim/internal/schema2/storage_qo_s.go | 17 - .../hcsshim/internal/schema2/storage_stats.go | 22 - .../hcsshim/internal/schema2/topology.go | 17 - .../hcsshim/internal/schema2/uefi.go | 21 - .../internal/schema2/uefi_boot_entry.go | 23 - .../hcsshim/internal/schema2/version.go | 17 - .../hcsshim/internal/schema2/video_monitor.go | 19 - .../internal/schema2/virtual_machine.go | 32 - .../internal/schema2/virtual_node_info.go | 21 - .../schema2/virtual_p_mem_controller.go | 20 - .../internal/schema2/virtual_p_mem_device.go | 19 - .../hcsshim/internal/schema2/virtual_smb.go | 17 - .../internal/schema2/virtual_smb_share.go | 21 - .../schema2/virtual_smb_share_options.go | 63 -- .../hcsshim/internal/schema2/vm_memory.go | 27 - .../schema2/windows_crash_reporting.go | 17 - .../hcsshim/internal/timeout/timeout.go | 70 -- .../hcsshim/internal/wclayer/activatelayer.go | 32 - .../hcsshim/internal/wclayer/baselayer.go | 173 ---- .../hcsshim/internal/wclayer/createlayer.go | 31 - .../internal/wclayer/createscratchlayer.go | 38 - .../internal/wclayer/deactivatelayer.go | 29 - .../hcsshim/internal/wclayer/destroylayer.go | 30 - .../internal/wclayer/expandscratchsize.go | 30 - .../hcsshim/internal/wclayer/exportlayer.go | 76 -- .../internal/wclayer/getlayermountpath.go | 56 -- .../internal/wclayer/getsharedbaseimages.go | 29 - .../hcsshim/internal/wclayer/grantvmaccess.go | 30 - .../hcsshim/internal/wclayer/importlayer.go | 135 --- .../hcsshim/internal/wclayer/layerexists.go | 33 - .../hcsshim/internal/wclayer/layerid.go | 13 - .../hcsshim/internal/wclayer/layerutils.go | 96 -- .../hcsshim/internal/wclayer/legacy.go | 815 ----------------- .../hcsshim/internal/wclayer/nametoguid.go | 34 - .../hcsshim/internal/wclayer/preparelayer.go | 47 - .../hcsshim/internal/wclayer/processimage.go | 23 - .../internal/wclayer/unpreparelayer.go | 30 - .../hcsshim/internal/wclayer/wclayer.go | 27 - .../internal/wclayer/zsyscall_windows.go | 510 ----------- vendor/github.com/Microsoft/hcsshim/layer.go | 106 --- .../github.com/Microsoft/hcsshim/process.go | 72 -- .../github.com/Microsoft/hcsshim/vendor.conf | 21 - .../Microsoft/hcsshim/zsyscall_windows.go | 54 -- .../archive/compression/compression.go | 21 + .../containerd/containerd/content/adaptor.go | 52 ++ .../containerd/containerd/content/content.go | 2 +- .../containerd/containerd/content/helpers.go | 74 +- .../containerd/content/local/locks.go | 13 +- .../containerd/content/local/readerat.go | 28 + .../containerd/content/local/store.go | 39 +- .../local/store_bsd.go} | 22 +- .../fds.go => content/local/store_openbsd.go} | 22 +- .../containerd/content/local/store_unix.go | 7 +- .../containerd/containerd/filters/filter.go | 1 - .../containerd/containerd/filters/parser.go | 1 - .../containerd/containerd/filters/quote.go | 8 +- .../containerd/containerd/images/diffid.go | 81 ++ .../containerd/containerd/images/image.go | 2 +- .../containerd/images/mediatypes.go | 53 +- .../labels.go} | 12 +- .../containerd/containerd/log/context.go | 14 +- .../containerd/platforms/compare.go | 166 ++-- .../containerd/platforms/cpuinfo.go | 35 +- .../containerd/platforms/defaults.go | 7 +- .../containerd/platforms/defaults_unix.go | 1 + .../containerd/platforms/defaults_windows.go | 63 +- .../containerd/platforms/platforms.go | 42 +- .../containerd/reference/reference.go | 4 + .../containerd/remotes/docker/auth/fetch.go | 209 +++++ .../remotes/docker/{auth.go => auth/parse.go} | 44 +- .../containerd/remotes/docker/authorizer.go | 292 ++---- .../containerd/remotes/docker/errcode.go | 2 +- .../containerd/remotes/docker/fetcher.go | 2 +- .../remotes/docker/httpreadseeker.go | 27 +- .../containerd/remotes/docker/pusher.go | 215 ++++- .../containerd/remotes/docker/registry.go | 43 +- .../containerd/remotes/docker/resolver.go | 47 +- .../containerd/remotes/docker/scope.go | 14 +- .../containerd/remotes/docker/status.go | 25 +- .../containerd/remotes/errors/errors.go | 56 ++ .../containerd/containerd/remotes/handlers.go | 41 +- .../containerd/containerd/remotes/resolver.go | 2 + .../containerd/containerd/sys/env.go | 33 - .../containerd/containerd/sys/epoll.go | 33 - .../containerd/containerd/sys/filesys.go | 35 - .../containerd/containerd/sys/filesys_unix.go | 31 - .../containerd/sys/filesys_windows.go | 268 ------ .../containerd/containerd/sys/mount_linux.go | 145 --- .../containerd/containerd/sys/oom_unix.go | 61 -- .../containerd/containerd/sys/oom_windows.go | 36 - .../containerd/containerd/sys/socket_unix.go | 80 -- .../containerd/containerd/sys/stat_bsd.go | 44 - .../containerd/containerd/sys/stat_unix.go | 44 - .../containerd/sys/subprocess_unsafe_linux.go | 30 - .../containerd/sys/subprocess_unsafe_linux.s | 15 - .../containerd/containerd/sys/userns_linux.go | 62 -- .../containerd/containerd/version/version.go | 2 +- vendor/github.com/klauspost/compress/LICENSE | 28 + .../klauspost/compress/fse/README.md | 79 ++ .../klauspost/compress/fse/bitreader.go | 107 +++ .../klauspost/compress/fse/bitwriter.go | 168 ++++ .../klauspost/compress/fse/bytereader.go | 56 ++ .../klauspost/compress/fse/compress.go | 684 ++++++++++++++ .../klauspost/compress/fse/decompress.go | 374 ++++++++ .../github.com/klauspost/compress/fse/fse.go | 143 +++ .../klauspost/compress/huff0/.gitignore | 1 + .../klauspost/compress/huff0/README.md | 87 ++ .../klauspost/compress/huff0/bitreader.go | 115 +++ .../klauspost/compress/huff0/bitwriter.go | 197 ++++ .../klauspost/compress/huff0/bytereader.go | 54 ++ .../klauspost/compress/huff0/compress.go | 625 +++++++++++++ .../klauspost/compress/huff0/decompress.go | 472 ++++++++++ .../klauspost/compress/huff0/huff0.go | 258 ++++++ .../klauspost/compress/snappy/.gitignore | 16 + .../klauspost/compress/snappy/AUTHORS | 15 + .../klauspost/compress/snappy/CONTRIBUTORS | 37 + .../klauspost/compress/snappy/LICENSE | 27 + .../klauspost/compress/snappy/README | 107 +++ .../klauspost/compress/snappy/decode.go | 237 +++++ .../klauspost/compress/snappy/decode_amd64.go | 14 + .../klauspost/compress/snappy/decode_amd64.s | 482 ++++++++++ .../klauspost/compress/snappy/decode_other.go | 115 +++ .../klauspost/compress/snappy/encode.go | 285 ++++++ .../klauspost/compress/snappy/encode_amd64.go | 29 + .../klauspost/compress/snappy/encode_amd64.s | 730 +++++++++++++++ .../klauspost/compress/snappy/encode_other.go | 238 +++++ .../klauspost/compress/snappy/runbench.cmd | 2 + .../klauspost/compress/snappy/snappy.go | 98 ++ .../klauspost/compress/zstd/README.md | 393 ++++++++ .../klauspost/compress/zstd/bitreader.go | 121 +++ .../klauspost/compress/zstd/bitwriter.go | 169 ++++ .../klauspost/compress/zstd/blockdec.go | 716 +++++++++++++++ .../klauspost/compress/zstd/blockenc.go | 845 ++++++++++++++++++ .../compress/zstd/blocktype_string.go | 85 ++ .../klauspost/compress/zstd/bytebuf.go | 127 +++ .../klauspost/compress/zstd/bytereader.go | 74 ++ .../klauspost/compress/zstd/decoder.go | 513 +++++++++++ .../compress/zstd/decoder_options.go | 68 ++ .../klauspost/compress/zstd/enc_dfast.go | 726 +++++++++++++++ .../klauspost/compress/zstd/enc_fast.go | 656 ++++++++++++++ .../klauspost/compress/zstd/enc_params.go | 154 ++++ .../klauspost/compress/zstd/encoder.go | 539 +++++++++++ .../compress/zstd/encoder_options.go | 231 +++++ .../klauspost/compress/zstd/framedec.go | 489 ++++++++++ .../klauspost/compress/zstd/frameenc.go | 115 +++ .../klauspost/compress/zstd/fse_decoder.go | 384 ++++++++ .../klauspost/compress/zstd/fse_encoder.go | 726 +++++++++++++++ .../klauspost/compress/zstd/fse_predefined.go | 158 ++++ .../klauspost/compress/zstd/hash.go | 77 ++ .../klauspost/compress/zstd/history.go | 73 ++ .../compress/zstd/internal/xxhash/LICENSE.txt | 22 + .../compress/zstd/internal/xxhash/README.md | 58 ++ .../compress/zstd/internal/xxhash/xxhash.go | 238 +++++ .../zstd/internal/xxhash/xxhash_amd64.go | 13 + .../zstd/internal/xxhash/xxhash_amd64.s | 215 +++++ .../zstd/internal/xxhash/xxhash_other.go | 76 ++ .../zstd/internal/xxhash/xxhash_safe.go | 11 + .../klauspost/compress/zstd/seqdec.go | 402 +++++++++ .../klauspost/compress/zstd/seqenc.go | 115 +++ .../klauspost/compress/zstd/snappy.go | 436 +++++++++ .../klauspost/compress/zstd/zstd.go | 136 +++ vendor/github.com/moby/locker/LICENSE | 190 ++++ vendor/github.com/moby/locker/README.md | 65 ++ vendor/github.com/moby/locker/locker.go | 112 +++ vendor/modules.txt | 73 +- 287 files changed, 17014 insertions(+), 11451 deletions(-) delete mode 100644 LICENSES/vendor/github.com/Microsoft/hcsshim/LICENSE create mode 100644 LICENSES/vendor/github.com/klauspost/compress/LICENSE create mode 100644 LICENSES/vendor/github.com/moby/locker/LICENSE delete mode 100644 vendor/github.com/Microsoft/hcsshim/.gitignore delete mode 100644 vendor/github.com/Microsoft/hcsshim/.gometalinter.json delete mode 100644 vendor/github.com/Microsoft/hcsshim/LICENSE delete mode 100644 vendor/github.com/Microsoft/hcsshim/README.md delete mode 100644 vendor/github.com/Microsoft/hcsshim/appveyor.yml delete mode 100644 vendor/github.com/Microsoft/hcsshim/container.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/errors.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/functional_tests.ps1 delete mode 100644 vendor/github.com/Microsoft/hcsshim/hcsshim.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/hnsendpoint.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/hnsglobals.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/hnsnetwork.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/hnspolicy.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/hnspolicylist.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/hnssupport.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/interface.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/guestrequest/types.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/guid/guid.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/hcs/callback.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/hcs/cgo.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/hcs/errors.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/hcs/hcs.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/hcs/log.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/hcs/process.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/hcs/system.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/hcs/utils.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/hcs/waithelper.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/hcs/watcher.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/hcs/zsyscall_windows.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/hcserror/hcserror.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/hns/hns.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/hns/hnsendpoint.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/hns/hnsfuncs.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/hns/hnsglobals.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/hns/hnsnetwork.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/hns/hnspolicy.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/hns/hnspolicylist.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/hns/hnssupport.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/hns/namespace.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/hns/zsyscall_windows.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/interop/interop.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/interop/zsyscall_windows.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/logfields/fields.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/longpath/longpath.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/mergemaps/merge.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/safefile/safeopen.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/safefile/zsyscall_windows.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema1/schema1.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/attachment.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/battery.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/cache_query_stats_response.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/chipset.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/close_handle.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/com_port.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/compute_system.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/configuration.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/console_size.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/container.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/container_credential_guard_state.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/container_memory_information.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/device.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/devices.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/enhanced_mode_video.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/flexible_io_device.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_connection.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_connection_info.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_crash_reporting.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_os.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_state.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/hosted_system.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket_2.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket_service_config.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket_system_config.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/keyboard.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/layer.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/linux_kernel_direct.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/mapped_directory.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/mapped_pipe.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/memory.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/memory_2.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/memory_information_for_vm.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/memory_stats.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/modify_setting_request.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/mouse.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/network_adapter.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/networking.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/pause_notification.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/pause_options.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/plan9.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/plan9_share.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/process_details.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/process_modify_request.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/process_parameters.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/process_status.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/processor.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/processor_2.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/processor_stats.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/properties.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/property_query.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/rdp_connection_options.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/registry_changes.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/registry_key.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/registry_value.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/restore_state.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/save_options.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/scsi.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/shared_memory_configuration.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/shared_memory_region.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/shared_memory_region_info.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/silo_properties.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/statistics.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/storage.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/storage_qo_s.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/storage_stats.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/topology.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/uefi.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/uefi_boot_entry.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/version.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/video_monitor.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_machine.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_node_info.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_p_mem_controller.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_p_mem_device.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_smb.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_smb_share.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_smb_share_options.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/vm_memory.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/schema2/windows_crash_reporting.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/timeout/timeout.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/wclayer/activatelayer.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/wclayer/baselayer.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/wclayer/createlayer.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/wclayer/createscratchlayer.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/wclayer/deactivatelayer.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/wclayer/destroylayer.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/wclayer/expandscratchsize.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/wclayer/exportlayer.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/wclayer/getlayermountpath.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/wclayer/getsharedbaseimages.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/wclayer/grantvmaccess.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/wclayer/importlayer.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/wclayer/layerexists.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/wclayer/layerid.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/wclayer/layerutils.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/wclayer/legacy.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/wclayer/nametoguid.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/wclayer/preparelayer.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/wclayer/processimage.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/wclayer/unpreparelayer.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/wclayer/wclayer.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/internal/wclayer/zsyscall_windows.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/layer.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/process.go delete mode 100644 vendor/github.com/Microsoft/hcsshim/vendor.conf delete mode 100644 vendor/github.com/Microsoft/hcsshim/zsyscall_windows.go create mode 100644 vendor/github.com/containerd/containerd/content/adaptor.go rename vendor/github.com/containerd/containerd/{sys/socket_windows.go => content/local/store_bsd.go} (61%) rename vendor/github.com/containerd/containerd/{sys/fds.go => content/local/store_openbsd.go} (64%) create mode 100644 vendor/github.com/containerd/containerd/images/diffid.go rename vendor/github.com/containerd/containerd/{sys/userns_unsupported.go => labels/labels.go} (76%) create mode 100644 vendor/github.com/containerd/containerd/remotes/docker/auth/fetch.go rename vendor/github.com/containerd/containerd/remotes/docker/{auth.go => auth/parse.go} (81%) create mode 100644 vendor/github.com/containerd/containerd/remotes/errors/errors.go delete mode 100644 vendor/github.com/containerd/containerd/sys/env.go delete mode 100644 vendor/github.com/containerd/containerd/sys/epoll.go delete mode 100644 vendor/github.com/containerd/containerd/sys/filesys.go delete mode 100644 vendor/github.com/containerd/containerd/sys/filesys_unix.go delete mode 100644 vendor/github.com/containerd/containerd/sys/filesys_windows.go delete mode 100644 vendor/github.com/containerd/containerd/sys/mount_linux.go delete mode 100644 vendor/github.com/containerd/containerd/sys/oom_unix.go delete mode 100644 vendor/github.com/containerd/containerd/sys/oom_windows.go delete mode 100644 vendor/github.com/containerd/containerd/sys/socket_unix.go delete mode 100644 vendor/github.com/containerd/containerd/sys/stat_bsd.go delete mode 100644 vendor/github.com/containerd/containerd/sys/stat_unix.go delete mode 100644 vendor/github.com/containerd/containerd/sys/subprocess_unsafe_linux.go delete mode 100644 vendor/github.com/containerd/containerd/sys/subprocess_unsafe_linux.s delete mode 100644 vendor/github.com/containerd/containerd/sys/userns_linux.go create mode 100644 vendor/github.com/klauspost/compress/LICENSE create mode 100644 vendor/github.com/klauspost/compress/fse/README.md create mode 100644 vendor/github.com/klauspost/compress/fse/bitreader.go create mode 100644 vendor/github.com/klauspost/compress/fse/bitwriter.go create mode 100644 vendor/github.com/klauspost/compress/fse/bytereader.go create mode 100644 vendor/github.com/klauspost/compress/fse/compress.go create mode 100644 vendor/github.com/klauspost/compress/fse/decompress.go create mode 100644 vendor/github.com/klauspost/compress/fse/fse.go create mode 100644 vendor/github.com/klauspost/compress/huff0/.gitignore create mode 100644 vendor/github.com/klauspost/compress/huff0/README.md create mode 100644 vendor/github.com/klauspost/compress/huff0/bitreader.go create mode 100644 vendor/github.com/klauspost/compress/huff0/bitwriter.go create mode 100644 vendor/github.com/klauspost/compress/huff0/bytereader.go create mode 100644 vendor/github.com/klauspost/compress/huff0/compress.go create mode 100644 vendor/github.com/klauspost/compress/huff0/decompress.go create mode 100644 vendor/github.com/klauspost/compress/huff0/huff0.go create mode 100644 vendor/github.com/klauspost/compress/snappy/.gitignore create mode 100644 vendor/github.com/klauspost/compress/snappy/AUTHORS create mode 100644 vendor/github.com/klauspost/compress/snappy/CONTRIBUTORS create mode 100644 vendor/github.com/klauspost/compress/snappy/LICENSE create mode 100644 vendor/github.com/klauspost/compress/snappy/README create mode 100644 vendor/github.com/klauspost/compress/snappy/decode.go create mode 100644 vendor/github.com/klauspost/compress/snappy/decode_amd64.go create mode 100644 vendor/github.com/klauspost/compress/snappy/decode_amd64.s create mode 100644 vendor/github.com/klauspost/compress/snappy/decode_other.go create mode 100644 vendor/github.com/klauspost/compress/snappy/encode.go create mode 100644 vendor/github.com/klauspost/compress/snappy/encode_amd64.go create mode 100644 vendor/github.com/klauspost/compress/snappy/encode_amd64.s create mode 100644 vendor/github.com/klauspost/compress/snappy/encode_other.go create mode 100644 vendor/github.com/klauspost/compress/snappy/runbench.cmd create mode 100644 vendor/github.com/klauspost/compress/snappy/snappy.go create mode 100644 vendor/github.com/klauspost/compress/zstd/README.md create mode 100644 vendor/github.com/klauspost/compress/zstd/bitreader.go create mode 100644 vendor/github.com/klauspost/compress/zstd/bitwriter.go create mode 100644 vendor/github.com/klauspost/compress/zstd/blockdec.go create mode 100644 vendor/github.com/klauspost/compress/zstd/blockenc.go create mode 100644 vendor/github.com/klauspost/compress/zstd/blocktype_string.go create mode 100644 vendor/github.com/klauspost/compress/zstd/bytebuf.go create mode 100644 vendor/github.com/klauspost/compress/zstd/bytereader.go create mode 100644 vendor/github.com/klauspost/compress/zstd/decoder.go create mode 100644 vendor/github.com/klauspost/compress/zstd/decoder_options.go create mode 100644 vendor/github.com/klauspost/compress/zstd/enc_dfast.go create mode 100644 vendor/github.com/klauspost/compress/zstd/enc_fast.go create mode 100644 vendor/github.com/klauspost/compress/zstd/enc_params.go create mode 100644 vendor/github.com/klauspost/compress/zstd/encoder.go create mode 100644 vendor/github.com/klauspost/compress/zstd/encoder_options.go create mode 100644 vendor/github.com/klauspost/compress/zstd/framedec.go create mode 100644 vendor/github.com/klauspost/compress/zstd/frameenc.go create mode 100644 vendor/github.com/klauspost/compress/zstd/fse_decoder.go create mode 100644 vendor/github.com/klauspost/compress/zstd/fse_encoder.go create mode 100644 vendor/github.com/klauspost/compress/zstd/fse_predefined.go create mode 100644 vendor/github.com/klauspost/compress/zstd/hash.go create mode 100644 vendor/github.com/klauspost/compress/zstd/history.go create mode 100644 vendor/github.com/klauspost/compress/zstd/internal/xxhash/LICENSE.txt create mode 100644 vendor/github.com/klauspost/compress/zstd/internal/xxhash/README.md create mode 100644 vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go create mode 100644 vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.go create mode 100644 vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s create mode 100644 vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go create mode 100644 vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_safe.go create mode 100644 vendor/github.com/klauspost/compress/zstd/seqdec.go create mode 100644 vendor/github.com/klauspost/compress/zstd/seqenc.go create mode 100644 vendor/github.com/klauspost/compress/zstd/snappy.go create mode 100644 vendor/github.com/klauspost/compress/zstd/zstd.go create mode 100644 vendor/github.com/moby/locker/LICENSE create mode 100644 vendor/github.com/moby/locker/README.md create mode 100644 vendor/github.com/moby/locker/locker.go diff --git a/LICENSES/vendor/github.com/Microsoft/hcsshim/LICENSE b/LICENSES/vendor/github.com/Microsoft/hcsshim/LICENSE deleted file mode 100644 index 075c7b3df..000000000 --- a/LICENSES/vendor/github.com/Microsoft/hcsshim/LICENSE +++ /dev/null @@ -1,24 +0,0 @@ -= vendor/github.com/Microsoft/hcsshim licensed under: = - -The MIT License (MIT) - -Copyright (c) 2015 Microsoft - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -= vendor/github.com/Microsoft/hcsshim/LICENSE d4c2cbbea5ee1e7c86dff68a7073718e diff --git a/LICENSES/vendor/github.com/klauspost/compress/LICENSE b/LICENSES/vendor/github.com/klauspost/compress/LICENSE new file mode 100644 index 000000000..c35f18a93 --- /dev/null +++ b/LICENSES/vendor/github.com/klauspost/compress/LICENSE @@ -0,0 +1,32 @@ += vendor/github.com/klauspost/compress licensed under: = + +Copyright (c) 2012 The Go Authors. All rights reserved. +Copyright (c) 2019 Klaus Post. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + += vendor/github.com/klauspost/compress/LICENSE f6eed50d75781660de81b193021f14a2 diff --git a/LICENSES/vendor/github.com/moby/locker/LICENSE b/LICENSES/vendor/github.com/moby/locker/LICENSE new file mode 100644 index 000000000..afdafb856 --- /dev/null +++ b/LICENSES/vendor/github.com/moby/locker/LICENSE @@ -0,0 +1,194 @@ += vendor/github.com/moby/locker licensed under: = + + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2013-2018 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + += vendor/github.com/moby/locker/LICENSE d4653096bbe6c7090ef605fe464c94e7 diff --git a/go.mod b/go.mod index e6c1702bb..70ba6c6c4 100644 --- a/go.mod +++ b/go.mod @@ -78,7 +78,7 @@ require ( golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 google.golang.org/grpc v1.49.0 gopkg.in/cas.v2 v2.2.0 - gopkg.in/square/go-jose.v2 v2.4.0 + gopkg.in/square/go-jose.v2 v2.5.1 gopkg.in/src-d/go-git.v4 v4.13.1 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 @@ -118,8 +118,7 @@ require ( github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/sprig/v3 v3.2.2 // indirect github.com/Masterminds/squirrel v1.5.3 // indirect - github.com/Microsoft/go-winio v0.5.1 // indirect - github.com/Microsoft/hcsshim v0.9.1 // indirect + github.com/Microsoft/go-winio v0.5.2 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/OneOfOne/xxhash v1.2.8 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect @@ -191,6 +190,7 @@ require ( github.com/jmoiron/sqlx v1.3.5 // indirect github.com/karrick/godirwalk v1.10.3 // indirect github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e // indirect + github.com/klauspost/compress v1.13.6 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect @@ -201,11 +201,12 @@ require ( github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-wordwrap v1.0.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/moby/locker v1.0.1 // indirect github.com/moby/spdystream v0.2.0 // indirect github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -221,7 +222,7 @@ require ( github.com/operator-framework/operator-lib v0.11.0 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pelletier/go-buffruneio v0.2.0 // indirect - github.com/pelletier/go-toml v1.7.0 // indirect + github.com/pelletier/go-toml v1.8.1 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/pquerna/cachecontrol v0.1.0 // indirect @@ -355,6 +356,7 @@ replace ( github.com/alecthomas/template => github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 github.com/alecthomas/units => github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d github.com/alessio/shellescape => github.com/alessio/shellescape v1.2.2 + github.com/alexflint/go-filemutex => github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae github.com/aliyun/aliyun-oss-go-sdk => github.com/aliyun/aliyun-oss-go-sdk v2.0.4+incompatible github.com/andreyvit/diff => github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 github.com/andybalholm/cascadia => github.com/andybalholm/cascadia v1.0.0 @@ -380,6 +382,7 @@ replace ( github.com/bgentry/speakeasy => github.com/bgentry/speakeasy v0.1.0 github.com/bitly/go-hostpool => github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 github.com/bitly/go-simplejson => github.com/bitly/go-simplejson v0.5.0 + github.com/bits-and-blooms/bitset => github.com/bits-and-blooms/bitset v1.2.0 github.com/blang/semver => github.com/blang/semver v3.5.0+incompatible github.com/blang/semver/v4 => github.com/blang/semver/v4 v4.0.0 github.com/bmizerany/assert => github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 @@ -388,6 +391,7 @@ replace ( github.com/bradfitz/gomemcache => github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b github.com/brancz/kube-rbac-proxy => github.com/brancz/kube-rbac-proxy v0.5.0 github.com/bshuster-repo/logrus-logstash-hook => github.com/bshuster-repo/logrus-logstash-hook v0.4.1 + github.com/buger/jsonparser => github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44 github.com/bugsnag/bugsnag-go => github.com/bugsnag/bugsnag-go v1.5.0 github.com/bugsnag/osext => github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b github.com/bugsnag/panicwrap => github.com/bugsnag/panicwrap v1.2.0 @@ -423,15 +427,28 @@ replace ( github.com/cockroachdb/datadriven => github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa github.com/codahale/hdrhistogram => github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd github.com/codegangsta/cli => github.com/codegangsta/cli v1.20.0 + github.com/containerd/aufs => github.com/containerd/aufs v1.0.0 + github.com/containerd/btrfs => github.com/containerd/btrfs v1.0.0 github.com/containerd/cgroups => github.com/containerd/cgroups v1.0.2 github.com/containerd/console => github.com/containerd/console v1.0.3 - github.com/containerd/containerd => github.com/containerd/containerd v1.4.13 + github.com/containerd/containerd => github.com/containerd/containerd v1.5.16 github.com/containerd/continuity => github.com/containerd/continuity v0.0.0-20181203112020-004b46473808 + github.com/containerd/fifo => github.com/containerd/fifo v1.0.0 + github.com/containerd/go-cni => github.com/containerd/go-cni v1.0.2 + github.com/containerd/go-runc => github.com/containerd/go-runc v1.0.0 + github.com/containerd/imgcrypt => github.com/containerd/imgcrypt v1.1.1 + github.com/containerd/nri => github.com/containerd/nri v0.1.0 github.com/containerd/stargz-snapshotter/estargz => github.com/containerd/stargz-snapshotter/estargz v0.7.0 + github.com/containerd/ttrpc => github.com/containerd/ttrpc v1.1.0 + github.com/containerd/typeurl => github.com/containerd/typeurl v1.0.2 + github.com/containerd/zfs => github.com/containerd/zfs v1.0.0 github.com/containernetworking/cni => github.com/containernetworking/cni v1.1.2 + github.com/containernetworking/plugins => github.com/containernetworking/plugins v0.9.1 + github.com/containers/ocicrypt => github.com/containers/ocicrypt v1.1.1 github.com/coreos/bbolt => github.com/coreos/bbolt v1.3.3 github.com/coreos/etcd => github.com/coreos/etcd v3.3.17+incompatible github.com/coreos/go-etcd => github.com/coreos/go-etcd v2.0.0+incompatible + github.com/coreos/go-iptables => github.com/coreos/go-iptables v0.5.0 github.com/coreos/go-oidc => github.com/coreos/go-oidc v2.1.0+incompatible github.com/coreos/go-semver => github.com/coreos/go-semver v0.3.0 github.com/coreos/go-systemd => github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f @@ -452,6 +469,10 @@ replace ( github.com/cznic/sortutil => github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 github.com/cznic/strutil => github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186 github.com/cznic/zappy => github.com/cznic/zappy v0.0.0-20160723133515-2533cb5b45cc + github.com/d2g/dhcp4 => github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c + github.com/d2g/dhcp4client => github.com/d2g/dhcp4client v1.0.0 + github.com/d2g/dhcp4server => github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5 + github.com/d2g/hardwareaddr => github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4 github.com/daaku/go.zipexe => github.com/daaku/go.zipexe v1.0.0 github.com/dave/jennifer => github.com/dave/jennifer v1.2.0 github.com/davecgh/go-spew => github.com/davecgh/go-spew v1.1.1 @@ -664,6 +685,7 @@ replace ( github.com/influxdata/roaring => github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6 github.com/influxdata/tdigest => github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9 github.com/influxdata/usage-client => github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368 + github.com/j-keck/arping => github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56 github.com/jackc/fake => github.com/jackc/fake v0.0.0-20150926172116-812a484cc733 github.com/jackc/pgx => github.com/jackc/pgx v3.2.0+incompatible github.com/jbenet/go-context => github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 @@ -741,11 +763,13 @@ replace ( github.com/mdlayher/wifi => github.com/mdlayher/wifi v0.0.0-20190303161829-b1436901ddee github.com/mgutz/ansi => github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b github.com/miekg/dns => github.com/miekg/dns v1.1.29 + github.com/miekg/pkcs11 => github.com/miekg/pkcs11 v1.0.3 github.com/mikefarah/yaml/v2 => github.com/mikefarah/yaml/v2 v2.4.0 github.com/mikefarah/yq/v2 => github.com/mikefarah/yq/v2 v2.4.1 github.com/minio/md5-simd => github.com/minio/md5-simd v1.1.0 github.com/minio/minio-go/v7 => github.com/minio/minio-go/v7 v7.0.2 github.com/minio/sha256-simd => github.com/minio/sha256-simd v0.1.1 + github.com/mistifyio/go-zfs => github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible github.com/mitchellh/cli => github.com/mitchellh/cli v1.0.0 github.com/mitchellh/copystructure => github.com/mitchellh/copystructure v1.0.0 github.com/mitchellh/go-homedir => github.com/mitchellh/go-homedir v1.1.0 @@ -758,6 +782,7 @@ replace ( github.com/moby/locker => github.com/moby/locker v1.0.1 github.com/moby/spdystream => github.com/moby/spdystream v0.2.0 github.com/moby/sys/mountinfo => github.com/moby/sys/mountinfo v0.5.0 + github.com/moby/sys/symlink => github.com/moby/sys/symlink v0.1.0 github.com/moby/term => github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 github.com/modern-go/concurrent => github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd github.com/modern-go/reflect2 => github.com/modern-go/reflect2 v1.0.2 @@ -858,6 +883,7 @@ replace ( github.com/russross/blackfriday => github.com/russross/blackfriday v1.5.2 github.com/russross/blackfriday/v2 => github.com/russross/blackfriday/v2 v2.1.0 github.com/ryanuber/columnize => github.com/ryanuber/columnize v2.1.0+incompatible + github.com/safchain/ethtool => github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8 github.com/samuel/go-zookeeper => github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da github.com/santhosh-tekuri/jsonschema => github.com/santhosh-tekuri/jsonschema v1.2.4 github.com/satori/go.uuid => github.com/satori/go.uuid v1.2.0 @@ -889,12 +915,14 @@ replace ( github.com/spf13/pflag => github.com/spf13/pflag v1.0.5 github.com/spf13/viper => github.com/spf13/viper v1.4.0 github.com/src-d/gcfg => github.com/src-d/gcfg v1.4.0 + github.com/stefanberger/go-pkcs11uri => github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980 github.com/stoewer/go-strcase => github.com/stoewer/go-strcase v1.2.0 github.com/streadway/amqp => github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271 github.com/streadway/handy => github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a github.com/stretchr/objx => github.com/stretchr/objx v0.2.0 github.com/stretchr/testify => github.com/stretchr/testify v1.4.0 github.com/syndtr/gocapability => github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 + github.com/tchap/go-patricia => github.com/tchap/go-patricia v2.2.6+incompatible github.com/tchap/go-patricia/v2 => github.com/tchap/go-patricia/v2 v2.3.1 github.com/thanos-io/thanos => github.com/thanos-io/thanos v0.13.1-0.20200910143741-e0b7f7b32e9c github.com/tidwall/pretty => github.com/tidwall/pretty v1.0.0 @@ -956,6 +984,7 @@ replace ( go.etcd.io/etcd/raft/v3 => go.etcd.io/etcd/raft/v3 v3.5.4 go.etcd.io/etcd/server/v3 => go.etcd.io/etcd/server/v3 v3.5.4 go.mongodb.org/mongo-driver => go.mongodb.org/mongo-driver v1.10.4 + go.mozilla.org/pkcs7 => go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1 go.opencensus.io => go.opencensus.io v0.22.3 go.opentelemetry.io/contrib => go.opentelemetry.io/contrib v0.20.0 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc => go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 @@ -1050,6 +1079,7 @@ replace ( k8s.io/code-generator => k8s.io/code-generator v0.25.3 k8s.io/component-base => k8s.io/component-base v0.25.3 k8s.io/component-helpers => k8s.io/component-helpers v0.25.3 + k8s.io/cri-api => k8s.io/cri-api v0.20.6 k8s.io/gengo => k8s.io/gengo v0.0.0-20211129171323-c02415ce4185 k8s.io/klog => k8s.io/klog v1.0.0 k8s.io/klog/v2 => k8s.io/klog/v2 v2.70.1 diff --git a/go.sum b/go.sum index 6f89f39c2..dd49843bb 100644 --- a/go.sum +++ b/go.sum @@ -69,6 +69,7 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alessio/shellescape v1.2.2/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= +github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= github.com/aliyun/aliyun-oss-go-sdk v2.0.4+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o= @@ -101,6 +102,7 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= @@ -110,6 +112,7 @@ github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx2 github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= github.com/brancz/kube-rbac-proxy v0.5.0/go.mod h1:cL2VjiIFGS90Cjh5ZZ8+It6tMcBt8rwvuw2J6Mamnl0= github.com/bshuster-repo/logrus-logstash-hook v0.4.1 h1:pgAtgj+A31JBVtEHu2uHuEx0n+2ukqUJnS2vVe5pQNA= +github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/bugsnag/bugsnag-go v1.5.0 h1:tP8hiPv1pGGW3LA6LKy5lW6WG+y9J2xWUdPd3WC452k= github.com/bugsnag/panicwrap v1.2.0 h1:OzrKrRvXis8qEvOkfcxNcYbOd2O7xXS2nnKMEMABFQA= github.com/bytecodealliance/wasmtime-go v1.0.0 h1:9u9gqaUiaJeN5IoD1L7egD8atOnTGyJcNp8BhkL9cUU= @@ -125,6 +128,7 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= +github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= github.com/chromedp/cdproto v0.0.0-20200424080200-0de008e41fa0 h1:Mf2aT0YmWsdNULwaHeCktDLWHb1s+VoDi9xEcFboLQ4= github.com/chromedp/cdproto v0.0.0-20200424080200-0de008e41fa0/go.mod h1:PfAWWKJqjlGFYJEidUM6aVIWPr0EpobeyVWEEmplX7g= github.com/chromedp/chromedp v0.5.3 h1:F9LafxmYpsQhWQBdCs+6Sret1zzeeFyHS5LkRF//Ffg= @@ -132,6 +136,7 @@ github.com/chromedp/chromedp v0.5.3/go.mod h1:YLdPtndaHQ4rCpSpBG+IPpy9JvX0VD+7aa github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= @@ -142,16 +147,31 @@ github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c/go.mod h1 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/containerd/containerd v1.4.13 h1:Z0CbagVdn9VN4K6htOCY/jApSw8YKP+RdLZ5dkXF8PM= -github.com/containerd/containerd v1.4.13/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/continuity v0.0.0-20181203112020-004b46473808 h1:4BX8f882bXEDKfWIf0wa8HRvpnBoPszJJXL+TVbBw4M= +github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= +github.com/containerd/btrfs v1.0.0/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= +github.com/containerd/cgroups v1.0.2/go.mod h1:qpbpJ1jmlqsR9f2IyaLPsdkCdnt0rbDVqIDlhuu5tRY= +github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= +github.com/containerd/containerd v1.5.16 h1:WsTS9tV0vQmRxkWAiiaoasHJ20jqVxVA15s93Bs4GIU= +github.com/containerd/containerd v1.5.16/go.mod h1:bVZZA+0blg2Lw6+I4xDml7L3gum0LsFKe3TnFELlSFw= +github.com/containerd/continuity v0.0.0-20181203112020-004b46473808/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= +github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk= +github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= +github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms= +github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= github.com/containerd/stargz-snapshotter/estargz v0.7.0 h1:1d/rydzTywc76lnjJb6qbPCiTiCwts49AzKps/Ecblw= github.com/containerd/stargz-snapshotter/estargz v0.7.0/go.mod h1:83VWDqHnurTKliEB0YvWMiCfLDwv4Cjj1X9Vk98GJZw= +github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ= +github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= +github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= github.com/containernetworking/cni v1.1.2 h1:wtRGZVv7olUHMOqouPpn3cXJWpJgM6+EUl31EQbXALQ= github.com/containernetworking/cni v1.1.2/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw= +github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8= +github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.17+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-oidc v2.1.0+incompatible h1:sdJrfw8akMnCuUlaZU3tE/uYXFgfqom8DBE9so9EBsM= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= @@ -177,6 +197,10 @@ github.com/cznic/ql v1.2.0/go.mod h1:FbpzhyZrqr0PVlK6ury+PoW3T0ODUV22OeWIxcaOrSE github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ= github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc= github.com/cznic/zappy v0.0.0-20160723133515-2533cb5b45cc/go.mod h1:Y1SNZ4dRUOKXshKUbwUapqNncRrho4mkjQebgEHZLj8= +github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= +github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= +github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= +github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -201,6 +225,7 @@ github.com/docker/engine v1.4.2-0.20200203170920-46ec8731fbce/go.mod h1:3CPr2caM github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-metrics v0.0.0-20181218153428-b84716841b82 h1:X0fj836zx99zFu83v/M79DuBn84IL/Syx1SY6Y5ZEMA= github.com/docker/go-metrics v0.0.0-20181218153428-b84716841b82/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= @@ -261,6 +286,7 @@ github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8 github.com/foxcpp/go-mockdns v0.0.0-20210729171921-fb145fc6f897 h1:E52jfcE64UG42SwLmrW0QByONfGynWuzBvm86BoB9z8= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsouza/fake-gcs-server v1.7.0/go.mod h1:5XIRs4YvwNbNoz+1JF8j6KLAyDh7RHGAyAK3EP2EsNk= @@ -467,6 +493,7 @@ github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19y github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= +github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= github.com/jackc/fake v0.0.0-20150926172116-812a484cc733/go.mod h1:WrMFNQdiFJ80sQsxDoMokWK1W5TQtxBFNpzWTD84ibQ= github.com/jackc/pgx v3.2.0+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= @@ -561,6 +588,7 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-oci8 v0.0.7/go.mod h1:wjDx6Xm9q7dFtHJvIlrI99JytznLw5wQ4R+9mNXJwGI= github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-shellwords v1.0.5/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= @@ -573,11 +601,13 @@ github.com/mdlayher/wifi v0.0.0-20190303161829-b1436901ddee/go.mod h1:Evt/EIne46 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.1.29 h1:xHBEhR+t5RzcFJjBLJlax2daXOrTYtr9z4WdKEfWFzg= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/mikefarah/yaml/v2 v2.4.0/go.mod h1:ahVqZF4n1W4NqwvVnZzC4es67xsW9uR/RRf2RRxieJU= github.com/mikefarah/yq/v2 v2.4.1/go.mod h1:i8SYf1XdgUvY2OFwSqGAtWOOgimD2McJ6iutoxRm4k0= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= github.com/minio/minio-go/v7 v7.0.2/go.mod h1:dJ80Mv2HeGkYLH1sqS/ksz07ON6csH3S6JUMSQ2zAns= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= @@ -591,8 +621,13 @@ github.com/mitchellh/mapstructure v1.2.2 h1:dxe5oCinTXiTIcfgmZecdCzPmAJKd46KsCWc github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= +github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/sys/mountinfo v0.5.0 h1:2Ks8/r6lopsxWi9m58nlwjaeSzUX9iiL1vj5qB/9ObI= +github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= +github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -606,6 +641,7 @@ github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c h1:nXxl5PrvVm2L/wCy8d github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mozillazg/go-cos v0.13.0/go.mod h1:Zp6DvvXn0RUOXGJ2chmWt2bLEqRAnJnS3DnAZsJsoaE= github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= +github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= @@ -643,6 +679,9 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/runc v1.1.4/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= +github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/opensearch-project/opensearch-go v1.1.0 h1:eG5sh3843bbU1itPRjA9QXbxcg8LaZ+DjEzQH9aLN3M= github.com/opensearch-project/opensearch-go v1.1.0/go.mod h1:+6/XHCuTH+fwsMJikZEWsucZ4eZMma3zNSeLrTtVGbo= github.com/opensearch-project/opensearch-go/v2 v2.0.0 h1:Ij3CpuHwey29cYPVMgi5h1pWBH2O0JaTXsa4c7pqhK4= @@ -734,10 +773,12 @@ github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNue github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/seccomp/libseccomp-golang v0.10.0/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/segmentio/fasthash v0.0.0-20180216231524-a72b379d632e/go.mod h1:tm/wZFQ8e24NYaBGIlnO2WGCAi67re4HHuOm0sftE/M= github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sercand/kuberesolver v2.4.0+incompatible/go.mod h1:lWF3GL0xptCB/vCiJPl/ZshwPsX/n4Y7u0CW9E7aQIQ= @@ -777,6 +818,7 @@ github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= +github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= @@ -784,6 +826,8 @@ github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= github.com/tchap/go-patricia/v2 v2.3.1 h1:6rQp39lgIYZ+MHmdEq4xzuk1t7OdC35z/xm0BGhTkes= github.com/tchap/go-patricia/v2 v2.3.1/go.mod h1:VZRHKAb53DLaG+nA9EaYYiaEx6YztwDlLElMsnSHD4k= github.com/thanos-io/thanos v0.13.1-0.20200910143741-e0b7f7b32e9c/go.mod h1:1IzeMKiS+pvxbG2M6ZJyi8ZHaAQKXNjDbP2gjhPbSXE= @@ -803,6 +847,8 @@ github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGr github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= +github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/weaveworks/common v0.0.0-20200820123129-280614068c5e/go.mod h1:hz10LOsAdzC3K/iXaKoFxOKTDRgxJl+BTGX1GY+TzO4= github.com/weaveworks/promrus v1.2.0/go.mod h1:SaE82+OJ91yqjrE1rsvBWVzNZKcHYFtMUyS1+Ogs/KA= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= @@ -858,6 +904,7 @@ go.etcd.io/etcd/server/v3 v3.5.4 h1:CMAZd0g8Bn5NRhynW6pKhc4FRg41/0QYy3d7aNm9874= go.etcd.io/etcd/server/v3 v3.5.4/go.mod h1:S5/YTU15KxymM5l3T6b09sNOHPXqGYIZStpuuGbb65c= go.mongodb.org/mongo-driver v1.10.4 h1:taPWsSsfn723M05lMyd/TAQe0kU9PsEYQ15WslnBtQw= go.mongodb.org/mongo-driver v1.10.4/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8= +go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opentelemetry.io/contrib v0.20.0 h1:ubFQUn0VCZ0gPwIoJfBJVpeBlyRMxu8Mm/huKWYd9p0= @@ -1011,6 +1058,7 @@ k8s.io/code-generator v0.25.3/go.mod h1:9F5fuVZOMWRme7MYj2YT3L9ropPWPokd9VRhVyD3 k8s.io/component-base v0.25.3 h1:UrsxciGdrCY03ULT1h/S/gXFCOPnLhUVwSyx+hM/zq4= k8s.io/component-base v0.25.3/go.mod h1:WYoS8L+IlTZgU7rhAl5Ctpw0WdMxDfCC5dkxcEFa/TI= k8s.io/component-helpers v0.25.3/go.mod h1:yu9zgPm9pf5jpmUzOZA9PMHY16Eu8ymt8AnSL0Xwbgw= +k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185 h1:TT1WdmqqXareKxZ/oNXEUSwKlLiHzPMyB0t8BaFeBYI= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= diff --git a/staging/src/kubesphere.io/api/go.mod b/staging/src/kubesphere.io/api/go.mod index 8a9f8b706..be6f7c42f 100644 --- a/staging/src/kubesphere.io/api/go.mod +++ b/staging/src/kubesphere.io/api/go.mod @@ -55,7 +55,7 @@ require ( github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/kr/pretty v0.2.1 // indirect github.com/mailru/easyjson v0.7.6 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect @@ -179,6 +179,7 @@ replace ( gonum.org/v1/netlib => gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 google.golang.org/appengine => google.golang.org/appengine v1.6.6 gopkg.in/check.v1 => gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 + gopkg.in/square/go-jose.v2 => gopkg.in/square/go-jose.v2 v2.4.0 gopkg.in/yaml.v3 => gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c k8s.io/client-go => k8s.io/client-go v0.25.3 kubesphere.io/api => ../api diff --git a/staging/src/kubesphere.io/api/go.sum b/staging/src/kubesphere.io/api/go.sum index ad4d05368..311765fd3 100644 --- a/staging/src/kubesphere.io/api/go.sum +++ b/staging/src/kubesphere.io/api/go.sum @@ -479,8 +479,6 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= @@ -705,7 +703,7 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.4.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tchap/go-patricia.v2 v2.2.6/go.mod h1:GjlIhdM7u6RWBtv58iEuqTR4NOShCtHo2EeySnNeNfs= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= diff --git a/staging/src/kubesphere.io/utils/go.mod b/staging/src/kubesphere.io/utils/go.mod index 865d6ad69..fc0f81610 100644 --- a/staging/src/kubesphere.io/utils/go.mod +++ b/staging/src/kubesphere.io/utils/go.mod @@ -24,7 +24,6 @@ require ( github.com/Masterminds/semver/v3 v3.1.1 // indirect github.com/Masterminds/sprig/v3 v3.2.2 // indirect github.com/Masterminds/squirrel v1.5.3 // indirect - github.com/Microsoft/go-winio v0.5.1 // indirect github.com/Microsoft/hcsshim v0.9.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect @@ -35,7 +34,6 @@ require ( github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect github.com/containerd/containerd v1.6.8 // indirect - github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7 // indirect github.com/cyphar/filepath-securejoin v0.2.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/cli v20.10.21+incompatible // indirect @@ -79,6 +77,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/karrick/godirwalk v1.10.3 // indirect + github.com/klauspost/compress v1.13.6 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/kr/pretty v0.2.1 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect @@ -87,11 +86,13 @@ require ( github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/mailru/easyjson v0.7.6 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-wordwrap v1.0.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/moby/locker v1.0.1 // indirect github.com/moby/spdystream v0.2.0 // indirect + github.com/moby/sys/mountinfo v0.5.0 // indirect github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -175,7 +176,9 @@ replace ( github.com/bshuster-repo/logrus-logstash-hook => github.com/bshuster-repo/logrus-logstash-hook v0.4.1 github.com/bugsnag/bugsnag-go => github.com/bugsnag/bugsnag-go v1.5.0 github.com/cespare/xxhash/v2 => github.com/cespare/xxhash/v2 v2.1.1 - github.com/containerd/containerd => github.com/containerd/containerd v1.4.13 + github.com/cilium/ebpf => github.com/cilium/ebpf v0.4.0 + github.com/containerd/cgroups => github.com/containerd/cgroups v1.0.2 + github.com/containerd/containerd => github.com/containerd/containerd v1.5.16 github.com/containerd/continuity => github.com/containerd/continuity v0.0.0-20181203112020-004b46473808 github.com/coreos/go-systemd => github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f github.com/cpuguy83/go-md2man/v2 => github.com/cpuguy83/go-md2man/v2 v2.0.0 @@ -198,6 +201,7 @@ replace ( github.com/gobuffalo/packr/v2 => github.com/gobuffalo/packr/v2 v2.2.0 github.com/gofrs/flock => github.com/gofrs/flock v0.7.1 github.com/gofrs/uuid => github.com/gofrs/uuid v3.2.0+incompatible + github.com/gogo/googleapis => github.com/gogo/googleapis v1.1.0 github.com/golang/groupcache => github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e github.com/golang/protobuf => github.com/golang/protobuf v1.4.2 github.com/golang/snappy => github.com/golang/snappy v0.0.1 @@ -241,6 +245,9 @@ replace ( github.com/onsi/ginkgo => github.com/onsi/ginkgo v1.14.0 github.com/onsi/gomega => github.com/onsi/gomega v1.10.1 github.com/opencontainers/image-spec => github.com/opencontainers/image-spec v1.0.1 + github.com/opencontainers/runtime-spec => github.com/opencontainers/runtime-spec v1.0.2 + github.com/opencontainers/selinux => github.com/opencontainers/selinux v1.10.0 + github.com/pelletier/go-toml => github.com/pelletier/go-toml v1.7.0 github.com/phayes/freeport => github.com/phayes/freeport v0.0.0-20171002181615-b8543db493a5 github.com/pquerna/cachecontrol => github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021 github.com/prometheus/client_golang => github.com/prometheus/client_golang v1.11.0 @@ -283,6 +290,7 @@ replace ( google.golang.org/api => google.golang.org/api v0.22.0 google.golang.org/appengine => google.golang.org/appengine v1.6.6 gopkg.in/check.v1 => gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 + gopkg.in/square/go-jose.v2 => gopkg.in/square/go-jose.v2 v2.4.0 gopkg.in/yaml.v3 => gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c k8s.io/client-go => k8s.io/client-go v0.25.3 kubesphere.io/utils => ../utils diff --git a/staging/src/kubesphere.io/utils/go.sum b/staging/src/kubesphere.io/utils/go.sum index f947203c6..3f755544d 100644 --- a/staging/src/kubesphere.io/utils/go.sum +++ b/staging/src/kubesphere.io/utils/go.sum @@ -24,7 +24,6 @@ github.com/Masterminds/sprig/v3 v3.0.0/go.mod h1:NEUY/Qq8Gdm2xgYA+NwJM6wmfdRV9xk github.com/Masterminds/squirrel v0.0.0-20161115235646-20f192218cf5 h1:PPfYWScYacO3Q6JMCLkyh6Ea2Q/REDTMgmiTAeiV8Jg= github.com/Masterminds/squirrel v0.0.0-20161115235646-20f192218cf5/go.mod h1:xnKTFzjGUiZtiOagBsfnvomW+nJg2usB1ZpordQWqNM= github.com/Microsoft/go-winio v0.4.12 h1:xAfWHN1IrQ0NJ9TBC0KBZoqLjzDTr1ML+4MywiUOryc= -github.com/Microsoft/go-winio v0.4.12/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/hcsshim v0.8.6 h1:ZfF0+zZeYdzMIVMZHKtDKJvLHj76XCuVae/jNkjj0IA= github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= @@ -78,10 +77,8 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/containerd/containerd v1.4.13 h1:Z0CbagVdn9VN4K6htOCY/jApSw8YKP+RdLZ5dkXF8PM= -github.com/containerd/containerd v1.4.13/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/continuity v0.0.0-20181203112020-004b46473808 h1:4BX8f882bXEDKfWIf0wa8HRvpnBoPszJJXL+TVbBw4M= -github.com/containerd/continuity v0.0.0-20181203112020-004b46473808/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/containerd v1.5.16 h1:WsTS9tV0vQmRxkWAiiaoasHJ20jqVxVA15s93Bs4GIU= +github.com/containerd/containerd v1.5.16/go.mod h1:bVZZA+0blg2Lw6+I4xDml7L3gum0LsFKe3TnFELlSFw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -290,6 +287,8 @@ github.com/karrick/godirwalk v1.10.3 h1:lOpSw2vJP0y5eLBW906QwKsUK/fe/QDyoqM5rnnu github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.9.5 h1:U+CaK85mrNNb4k8BNOfgJtJ/gr6kswUCFj6miSzVC6M= +github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -339,8 +338,12 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= +github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/sys/mountinfo v0.5.0 h1:2Ks8/r6lopsxWi9m58nlwjaeSzUX9iiL1vj5qB/9ObI= +github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -388,7 +391,7 @@ github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnh github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= diff --git a/vendor/github.com/Microsoft/hcsshim/.gitignore b/vendor/github.com/Microsoft/hcsshim/.gitignore deleted file mode 100644 index b883f1fdc..000000000 --- a/vendor/github.com/Microsoft/hcsshim/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.exe diff --git a/vendor/github.com/Microsoft/hcsshim/.gometalinter.json b/vendor/github.com/Microsoft/hcsshim/.gometalinter.json deleted file mode 100644 index 00e9a6e2e..000000000 --- a/vendor/github.com/Microsoft/hcsshim/.gometalinter.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "Vendor": true, - "Deadline": "2m", - "Sort": [ - "linter", - "severity", - "path", - "line" - ], - "Skip": [ - "internal\\schema2" - ], - "EnableGC": true, - "Enable": [ - "gofmt" - ] -} \ No newline at end of file diff --git a/vendor/github.com/Microsoft/hcsshim/LICENSE b/vendor/github.com/Microsoft/hcsshim/LICENSE deleted file mode 100644 index 49d21669a..000000000 --- a/vendor/github.com/Microsoft/hcsshim/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2015 Microsoft - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/vendor/github.com/Microsoft/hcsshim/README.md b/vendor/github.com/Microsoft/hcsshim/README.md deleted file mode 100644 index 15b39181a..000000000 --- a/vendor/github.com/Microsoft/hcsshim/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# hcsshim - -[![Build status](https://ci.appveyor.com/api/projects/status/nbcw28mnkqml0loa/branch/master?svg=true)](https://ci.appveyor.com/project/WindowsVirtualization/hcsshim/branch/master) - -This package contains the Golang interface for using the Windows [Host Compute Service](https://blogs.technet.microsoft.com/virtualization/2017/01/27/introducing-the-host-compute-service-hcs/) (HCS) to launch and manage [Windows Containers](https://docs.microsoft.com/en-us/virtualization/windowscontainers/about/). It also contains other helpers and functions for managing Windows Containers such as the Golang interface for the Host Network Service (HNS). - -It is primarily used in the [Moby Project](https://github.com/moby/moby), but it can be freely used by other projects as well. - -## Contributing - -This project welcomes contributions and suggestions. Most contributions require you to agree to a -Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us -the rights to use your contribution. For details, visit https://cla.microsoft.com. - -When you submit a pull request, a CLA-bot will automatically determine whether you need to provide -a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions -provided by the bot. You will only need to do this once across all repos using our CLA. - -This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). -For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or -contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. - -## Dependencies - -This project requires Golang 1.9 or newer to build. - -For system requirements to run this project, see the Microsoft docs on [Windows Container requirements](https://docs.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/system-requirements). - -## Reporting Security Issues - -Security issues and bugs should be reported privately, via email, to the Microsoft Security -Response Center (MSRC) at [secure@microsoft.com](mailto:secure@microsoft.com). You should -receive a response within 24 hours. If for some reason you do not, please follow up via -email to ensure we received your original message. Further information, including the -[MSRC PGP](https://technet.microsoft.com/en-us/security/dn606155) key, can be found in -the [Security TechCenter](https://technet.microsoft.com/en-us/security/default). - -For additional details, see [Report a Computer Security Vulnerability](https://technet.microsoft.com/en-us/security/ff852094.aspx) on Technet - ---------------- -Copyright (c) 2018 Microsoft Corp. All rights reserved. diff --git a/vendor/github.com/Microsoft/hcsshim/appveyor.yml b/vendor/github.com/Microsoft/hcsshim/appveyor.yml deleted file mode 100644 index a8ec5a593..000000000 --- a/vendor/github.com/Microsoft/hcsshim/appveyor.yml +++ /dev/null @@ -1,29 +0,0 @@ -version: 0.1.{build} - -image: Visual Studio 2017 - -clone_folder: c:\gopath\src\github.com\Microsoft\hcsshim - -environment: - GOPATH: c:\gopath - PATH: C:\mingw-w64\x86_64-7.2.0-posix-seh-rt_v5-rev1\mingw64\bin;%GOPATH%\bin;C:\gometalinter-2.0.12-windows-amd64;%PATH% - -stack: go 1.11 - -build_script: - - appveyor DownloadFile https://github.com/alecthomas/gometalinter/releases/download/v2.0.12/gometalinter-2.0.12-windows-amd64.zip - - 7z x gometalinter-2.0.12-windows-amd64.zip -y -oC:\ > NUL - - gometalinter.exe --config .gometalinter.json ./... - - go build ./cmd/wclayer - - go build ./cmd/runhcs - - go build ./cmd/tar2ext4 - - go test -v ./... -tags admin - - go test -c ./test/functional/ -tags functional - - go test -c ./test/runhcs/ -tags integration - -artifacts: - - path: 'wclayer.exe' - - path: 'runhcs.exe' - - path: 'tar2ext4.exe' - - path: 'functional.test.exe' - - path: 'runhcs.test.exe' \ No newline at end of file diff --git a/vendor/github.com/Microsoft/hcsshim/container.go b/vendor/github.com/Microsoft/hcsshim/container.go deleted file mode 100644 index e142c3154..000000000 --- a/vendor/github.com/Microsoft/hcsshim/container.go +++ /dev/null @@ -1,192 +0,0 @@ -package hcsshim - -import ( - "fmt" - "os" - "time" - - "github.com/Microsoft/hcsshim/internal/hcs" - "github.com/Microsoft/hcsshim/internal/mergemaps" - "github.com/Microsoft/hcsshim/internal/schema1" -) - -// ContainerProperties holds the properties for a container and the processes running in that container -type ContainerProperties = schema1.ContainerProperties - -// MemoryStats holds the memory statistics for a container -type MemoryStats = schema1.MemoryStats - -// ProcessorStats holds the processor statistics for a container -type ProcessorStats = schema1.ProcessorStats - -// StorageStats holds the storage statistics for a container -type StorageStats = schema1.StorageStats - -// NetworkStats holds the network statistics for a container -type NetworkStats = schema1.NetworkStats - -// Statistics is the structure returned by a statistics call on a container -type Statistics = schema1.Statistics - -// ProcessList is the structure of an item returned by a ProcessList call on a container -type ProcessListItem = schema1.ProcessListItem - -// MappedVirtualDiskController is the structure of an item returned by a MappedVirtualDiskList call on a container -type MappedVirtualDiskController = schema1.MappedVirtualDiskController - -// Type of Request Support in ModifySystem -type RequestType = schema1.RequestType - -// Type of Resource Support in ModifySystem -type ResourceType = schema1.ResourceType - -// RequestType const -const ( - Add = schema1.Add - Remove = schema1.Remove - Network = schema1.Network -) - -// ResourceModificationRequestResponse is the structure used to send request to the container to modify the system -// Supported resource types are Network and Request Types are Add/Remove -type ResourceModificationRequestResponse = schema1.ResourceModificationRequestResponse - -type container struct { - system *hcs.System -} - -// createComputeSystemAdditionalJSON is read from the environment at initialisation -// time. It allows an environment variable to define additional JSON which -// is merged in the CreateComputeSystem call to HCS. -var createContainerAdditionalJSON []byte - -func init() { - createContainerAdditionalJSON = ([]byte)(os.Getenv("HCSSHIM_CREATECONTAINER_ADDITIONALJSON")) -} - -// CreateContainer creates a new container with the given configuration but does not start it. -func CreateContainer(id string, c *ContainerConfig) (Container, error) { - fullConfig, err := mergemaps.MergeJSON(c, createContainerAdditionalJSON) - if err != nil { - return nil, fmt.Errorf("failed to merge additional JSON '%s': %s", createContainerAdditionalJSON, err) - } - - system, err := hcs.CreateComputeSystem(id, fullConfig) - if err != nil { - return nil, err - } - return &container{system}, err -} - -// OpenContainer opens an existing container by ID. -func OpenContainer(id string) (Container, error) { - system, err := hcs.OpenComputeSystem(id) - if err != nil { - return nil, err - } - return &container{system}, err -} - -// GetContainers gets a list of the containers on the system that match the query -func GetContainers(q ComputeSystemQuery) ([]ContainerProperties, error) { - return hcs.GetComputeSystems(q) -} - -// Start synchronously starts the container. -func (container *container) Start() error { - return convertSystemError(container.system.Start(), container) -} - -// Shutdown requests a container shutdown, but it may not actually be shutdown until Wait() succeeds. -func (container *container) Shutdown() error { - return convertSystemError(container.system.Shutdown(), container) -} - -// Terminate requests a container terminate, but it may not actually be terminated until Wait() succeeds. -func (container *container) Terminate() error { - return convertSystemError(container.system.Terminate(), container) -} - -// Waits synchronously waits for the container to shutdown or terminate. -func (container *container) Wait() error { - return convertSystemError(container.system.Wait(), container) -} - -// WaitTimeout synchronously waits for the container to terminate or the duration to elapse. It -// returns false if timeout occurs. -func (container *container) WaitTimeout(t time.Duration) error { - return convertSystemError(container.system.WaitTimeout(t), container) -} - -// Pause pauses the execution of a container. -func (container *container) Pause() error { - return convertSystemError(container.system.Pause(), container) -} - -// Resume resumes the execution of a container. -func (container *container) Resume() error { - return convertSystemError(container.system.Resume(), container) -} - -// HasPendingUpdates returns true if the container has updates pending to install -func (container *container) HasPendingUpdates() (bool, error) { - return false, nil -} - -// Statistics returns statistics for the container. This is a legacy v1 call -func (container *container) Statistics() (Statistics, error) { - properties, err := container.system.Properties(schema1.PropertyTypeStatistics) - if err != nil { - return Statistics{}, convertSystemError(err, container) - } - - return properties.Statistics, nil -} - -// ProcessList returns an array of ProcessListItems for the container. This is a legacy v1 call -func (container *container) ProcessList() ([]ProcessListItem, error) { - properties, err := container.system.Properties(schema1.PropertyTypeProcessList) - if err != nil { - return nil, convertSystemError(err, container) - } - - return properties.ProcessList, nil -} - -// This is a legacy v1 call -func (container *container) MappedVirtualDisks() (map[int]MappedVirtualDiskController, error) { - properties, err := container.system.Properties(schema1.PropertyTypeMappedVirtualDisk) - if err != nil { - return nil, convertSystemError(err, container) - } - - return properties.MappedVirtualDiskControllers, nil -} - -// CreateProcess launches a new process within the container. -func (container *container) CreateProcess(c *ProcessConfig) (Process, error) { - p, err := container.system.CreateProcess(c) - if err != nil { - return nil, convertSystemError(err, container) - } - return &process{p}, nil -} - -// OpenProcess gets an interface to an existing process within the container. -func (container *container) OpenProcess(pid int) (Process, error) { - p, err := container.system.OpenProcess(pid) - if err != nil { - return nil, convertSystemError(err, container) - } - return &process{p}, nil -} - -// Close cleans up any state associated with the container but does not terminate or wait for it. -func (container *container) Close() error { - return convertSystemError(container.system.Close(), container) -} - -// Modify the System -func (container *container) Modify(config *ResourceModificationRequestResponse) error { - return convertSystemError(container.system.Modify(config), container) -} diff --git a/vendor/github.com/Microsoft/hcsshim/errors.go b/vendor/github.com/Microsoft/hcsshim/errors.go deleted file mode 100644 index 63efa23c7..000000000 --- a/vendor/github.com/Microsoft/hcsshim/errors.go +++ /dev/null @@ -1,257 +0,0 @@ -package hcsshim - -import ( - "fmt" - "syscall" - - "github.com/Microsoft/hcsshim/internal/hns" - - "github.com/Microsoft/hcsshim/internal/hcs" - "github.com/Microsoft/hcsshim/internal/hcserror" -) - -var ( - // ErrComputeSystemDoesNotExist is an error encountered when the container being operated on no longer exists = hcs.exist - ErrComputeSystemDoesNotExist = hcs.ErrComputeSystemDoesNotExist - - // ErrElementNotFound is an error encountered when the object being referenced does not exist - ErrElementNotFound = hcs.ErrElementNotFound - - // ErrElementNotFound is an error encountered when the object being referenced does not exist - ErrNotSupported = hcs.ErrNotSupported - - // ErrInvalidData is an error encountered when the request being sent to hcs is invalid/unsupported - // decimal -2147024883 / hex 0x8007000d - ErrInvalidData = hcs.ErrInvalidData - - // ErrHandleClose is an error encountered when the handle generating the notification being waited on has been closed - ErrHandleClose = hcs.ErrHandleClose - - // ErrAlreadyClosed is an error encountered when using a handle that has been closed by the Close method - ErrAlreadyClosed = hcs.ErrAlreadyClosed - - // ErrInvalidNotificationType is an error encountered when an invalid notification type is used - ErrInvalidNotificationType = hcs.ErrInvalidNotificationType - - // ErrInvalidProcessState is an error encountered when the process is not in a valid state for the requested operation - ErrInvalidProcessState = hcs.ErrInvalidProcessState - - // ErrTimeout is an error encountered when waiting on a notification times out - ErrTimeout = hcs.ErrTimeout - - // ErrUnexpectedContainerExit is the error encountered when a container exits while waiting for - // a different expected notification - ErrUnexpectedContainerExit = hcs.ErrUnexpectedContainerExit - - // ErrUnexpectedProcessAbort is the error encountered when communication with the compute service - // is lost while waiting for a notification - ErrUnexpectedProcessAbort = hcs.ErrUnexpectedProcessAbort - - // ErrUnexpectedValue is an error encountered when hcs returns an invalid value - ErrUnexpectedValue = hcs.ErrUnexpectedValue - - // ErrVmcomputeAlreadyStopped is an error encountered when a shutdown or terminate request is made on a stopped container - ErrVmcomputeAlreadyStopped = hcs.ErrVmcomputeAlreadyStopped - - // ErrVmcomputeOperationPending is an error encountered when the operation is being completed asynchronously - ErrVmcomputeOperationPending = hcs.ErrVmcomputeOperationPending - - // ErrVmcomputeOperationInvalidState is an error encountered when the compute system is not in a valid state for the requested operation - ErrVmcomputeOperationInvalidState = hcs.ErrVmcomputeOperationInvalidState - - // ErrProcNotFound is an error encountered when the the process cannot be found - ErrProcNotFound = hcs.ErrProcNotFound - - // ErrVmcomputeOperationAccessIsDenied is an error which can be encountered when enumerating compute systems in RS1/RS2 - // builds when the underlying silo might be in the process of terminating. HCS was fixed in RS3. - ErrVmcomputeOperationAccessIsDenied = hcs.ErrVmcomputeOperationAccessIsDenied - - // ErrVmcomputeInvalidJSON is an error encountered when the compute system does not support/understand the messages sent by management - ErrVmcomputeInvalidJSON = hcs.ErrVmcomputeInvalidJSON - - // ErrVmcomputeUnknownMessage is an error encountered guest compute system doesn't support the message - ErrVmcomputeUnknownMessage = hcs.ErrVmcomputeUnknownMessage - - // ErrNotSupported is an error encountered when hcs doesn't support the request - ErrPlatformNotSupported = hcs.ErrPlatformNotSupported -) - -type EndpointNotFoundError = hns.EndpointNotFoundError -type NetworkNotFoundError = hns.NetworkNotFoundError - -// ProcessError is an error encountered in HCS during an operation on a Process object -type ProcessError struct { - Process *process - Operation string - ExtraInfo string - Err error - Events []hcs.ErrorEvent -} - -// ContainerError is an error encountered in HCS during an operation on a Container object -type ContainerError struct { - Container *container - Operation string - ExtraInfo string - Err error - Events []hcs.ErrorEvent -} - -func (e *ContainerError) Error() string { - if e == nil { - return "" - } - - if e.Container == nil { - return "unexpected nil container for error: " + e.Err.Error() - } - - s := "container " + e.Container.system.ID() - - if e.Operation != "" { - s += " encountered an error during " + e.Operation - } - - switch e.Err.(type) { - case nil: - break - case syscall.Errno: - s += fmt.Sprintf(": failure in a Windows system call: %s (0x%x)", e.Err, hcserror.Win32FromError(e.Err)) - default: - s += fmt.Sprintf(": %s", e.Err.Error()) - } - - for _, ev := range e.Events { - s += "\n" + ev.String() - } - - if e.ExtraInfo != "" { - s += " extra info: " + e.ExtraInfo - } - - return s -} - -func makeContainerError(container *container, operation string, extraInfo string, err error) error { - // Don't double wrap errors - if _, ok := err.(*ContainerError); ok { - return err - } - containerError := &ContainerError{Container: container, Operation: operation, ExtraInfo: extraInfo, Err: err} - return containerError -} - -func (e *ProcessError) Error() string { - if e == nil { - return "" - } - - if e.Process == nil { - return "Unexpected nil process for error: " + e.Err.Error() - } - - s := fmt.Sprintf("process %d in container %s", e.Process.p.Pid(), e.Process.p.SystemID()) - if e.Operation != "" { - s += " encountered an error during " + e.Operation - } - - switch e.Err.(type) { - case nil: - break - case syscall.Errno: - s += fmt.Sprintf(": failure in a Windows system call: %s (0x%x)", e.Err, hcserror.Win32FromError(e.Err)) - default: - s += fmt.Sprintf(": %s", e.Err.Error()) - } - - for _, ev := range e.Events { - s += "\n" + ev.String() - } - - return s -} - -func makeProcessError(process *process, operation string, extraInfo string, err error) error { - // Don't double wrap errors - if _, ok := err.(*ProcessError); ok { - return err - } - processError := &ProcessError{Process: process, Operation: operation, ExtraInfo: extraInfo, Err: err} - return processError -} - -// IsNotExist checks if an error is caused by the Container or Process not existing. -// Note: Currently, ErrElementNotFound can mean that a Process has either -// already exited, or does not exist. Both IsAlreadyStopped and IsNotExist -// will currently return true when the error is ErrElementNotFound or ErrProcNotFound. -func IsNotExist(err error) bool { - if _, ok := err.(EndpointNotFoundError); ok { - return true - } - if _, ok := err.(NetworkNotFoundError); ok { - return true - } - return hcs.IsNotExist(getInnerError(err)) -} - -// IsAlreadyClosed checks if an error is caused by the Container or Process having been -// already closed by a call to the Close() method. -func IsAlreadyClosed(err error) bool { - return hcs.IsAlreadyClosed(getInnerError(err)) -} - -// IsPending returns a boolean indicating whether the error is that -// the requested operation is being completed in the background. -func IsPending(err error) bool { - return hcs.IsPending(getInnerError(err)) -} - -// IsTimeout returns a boolean indicating whether the error is caused by -// a timeout waiting for the operation to complete. -func IsTimeout(err error) bool { - return hcs.IsTimeout(getInnerError(err)) -} - -// IsAlreadyStopped returns a boolean indicating whether the error is caused by -// a Container or Process being already stopped. -// Note: Currently, ErrElementNotFound can mean that a Process has either -// already exited, or does not exist. Both IsAlreadyStopped and IsNotExist -// will currently return true when the error is ErrElementNotFound or ErrProcNotFound. -func IsAlreadyStopped(err error) bool { - return hcs.IsAlreadyStopped(getInnerError(err)) -} - -// IsNotSupported returns a boolean indicating whether the error is caused by -// unsupported platform requests -// Note: Currently Unsupported platform requests can be mean either -// ErrVmcomputeInvalidJSON, ErrInvalidData, ErrNotSupported or ErrVmcomputeUnknownMessage -// is thrown from the Platform -func IsNotSupported(err error) bool { - return hcs.IsNotSupported(getInnerError(err)) -} - -func getInnerError(err error) error { - switch pe := err.(type) { - case nil: - return nil - case *ContainerError: - err = pe.Err - case *ProcessError: - err = pe.Err - } - return err -} - -func convertSystemError(err error, c *container) error { - if serr, ok := err.(*hcs.SystemError); ok { - return &ContainerError{Container: c, Operation: serr.Op, ExtraInfo: serr.Extra, Err: serr.Err, Events: serr.Events} - } - return err -} - -func convertProcessError(err error, p *process) error { - if perr, ok := err.(*hcs.ProcessError); ok { - return &ProcessError{Process: p, Operation: perr.Op, Err: perr.Err, Events: perr.Events} - } - return err -} diff --git a/vendor/github.com/Microsoft/hcsshim/functional_tests.ps1 b/vendor/github.com/Microsoft/hcsshim/functional_tests.ps1 deleted file mode 100644 index ce6edbcf3..000000000 --- a/vendor/github.com/Microsoft/hcsshim/functional_tests.ps1 +++ /dev/null @@ -1,12 +0,0 @@ -# Requirements so far: -# dockerd running -# - image microsoft/nanoserver (matching host base image) docker load -i c:\baseimages\nanoserver.tar -# - image alpine (linux) docker pull --platform=linux alpine - - -# TODO: Add this a parameter for debugging. ie "functional-tests -debug=$true" -#$env:HCSSHIM_FUNCTIONAL_TESTS_DEBUG="yes please" - -#pushd uvm -go test -v -tags "functional uvmcreate uvmscratch uvmscsi uvmvpmem uvmvsmb uvmp9" ./... -#popd \ No newline at end of file diff --git a/vendor/github.com/Microsoft/hcsshim/hcsshim.go b/vendor/github.com/Microsoft/hcsshim/hcsshim.go deleted file mode 100644 index ceb3ac85e..000000000 --- a/vendor/github.com/Microsoft/hcsshim/hcsshim.go +++ /dev/null @@ -1,28 +0,0 @@ -// Shim for the Host Compute Service (HCS) to manage Windows Server -// containers and Hyper-V containers. - -package hcsshim - -import ( - "syscall" - - "github.com/Microsoft/hcsshim/internal/hcserror" -) - -//go:generate go run mksyscall_windows.go -output zsyscall_windows.go hcsshim.go - -//sys SetCurrentThreadCompartmentId(compartmentId uint32) (hr error) = iphlpapi.SetCurrentThreadCompartmentId - -const ( - // Specific user-visible exit codes - WaitErrExecFailed = 32767 - - ERROR_GEN_FAILURE = hcserror.ERROR_GEN_FAILURE - ERROR_SHUTDOWN_IN_PROGRESS = syscall.Errno(1115) - WSAEINVAL = syscall.Errno(10022) - - // Timeout on wait calls - TimeoutInfinite = 0xFFFFFFFF -) - -type HcsError = hcserror.HcsError diff --git a/vendor/github.com/Microsoft/hcsshim/hnsendpoint.go b/vendor/github.com/Microsoft/hcsshim/hnsendpoint.go deleted file mode 100644 index eb013d2c4..000000000 --- a/vendor/github.com/Microsoft/hcsshim/hnsendpoint.go +++ /dev/null @@ -1,94 +0,0 @@ -package hcsshim - -import ( - "github.com/Microsoft/hcsshim/internal/hns" -) - -// HNSEndpoint represents a network endpoint in HNS -type HNSEndpoint = hns.HNSEndpoint - -// Namespace represents a Compartment. -type Namespace = hns.Namespace - -//SystemType represents the type of the system on which actions are done -type SystemType string - -// SystemType const -const ( - ContainerType SystemType = "Container" - VirtualMachineType SystemType = "VirtualMachine" - HostType SystemType = "Host" -) - -// EndpointAttachDetachRequest is the structure used to send request to the container to modify the system -// Supported resource types are Network and Request Types are Add/Remove -type EndpointAttachDetachRequest = hns.EndpointAttachDetachRequest - -// EndpointResquestResponse is object to get the endpoint request response -type EndpointResquestResponse = hns.EndpointResquestResponse - -// HNSEndpointRequest makes a HNS call to modify/query a network endpoint -func HNSEndpointRequest(method, path, request string) (*HNSEndpoint, error) { - return hns.HNSEndpointRequest(method, path, request) -} - -// HNSListEndpointRequest makes a HNS call to query the list of available endpoints -func HNSListEndpointRequest() ([]HNSEndpoint, error) { - return hns.HNSListEndpointRequest() -} - -// HotAttachEndpoint makes a HCS Call to attach the endpoint to the container -func HotAttachEndpoint(containerID string, endpointID string) error { - return modifyNetworkEndpoint(containerID, endpointID, Add) -} - -// HotDetachEndpoint makes a HCS Call to detach the endpoint from the container -func HotDetachEndpoint(containerID string, endpointID string) error { - return modifyNetworkEndpoint(containerID, endpointID, Remove) -} - -// ModifyContainer corresponding to the container id, by sending a request -func modifyContainer(id string, request *ResourceModificationRequestResponse) error { - container, err := OpenContainer(id) - if err != nil { - if IsNotExist(err) { - return ErrComputeSystemDoesNotExist - } - return getInnerError(err) - } - defer container.Close() - err = container.Modify(request) - if err != nil { - if IsNotSupported(err) { - return ErrPlatformNotSupported - } - return getInnerError(err) - } - - return nil -} - -func modifyNetworkEndpoint(containerID string, endpointID string, request RequestType) error { - requestMessage := &ResourceModificationRequestResponse{ - Resource: Network, - Request: request, - Data: endpointID, - } - err := modifyContainer(containerID, requestMessage) - - if err != nil { - return err - } - - return nil -} - -// GetHNSEndpointByID get the Endpoint by ID -func GetHNSEndpointByID(endpointID string) (*HNSEndpoint, error) { - return hns.GetHNSEndpointByID(endpointID) -} - -// GetHNSEndpointByName gets the endpoint filtered by Name -func GetHNSEndpointByName(endpointName string) (*HNSEndpoint, error) { - return hns.GetHNSEndpointByName(endpointName) -} diff --git a/vendor/github.com/Microsoft/hcsshim/hnsglobals.go b/vendor/github.com/Microsoft/hcsshim/hnsglobals.go deleted file mode 100644 index 2b5381904..000000000 --- a/vendor/github.com/Microsoft/hcsshim/hnsglobals.go +++ /dev/null @@ -1,16 +0,0 @@ -package hcsshim - -import ( - "github.com/Microsoft/hcsshim/internal/hns" -) - -type HNSGlobals = hns.HNSGlobals -type HNSVersion = hns.HNSVersion - -var ( - HNSVersion1803 = hns.HNSVersion1803 -) - -func GetHNSGlobals() (*HNSGlobals, error) { - return hns.GetHNSGlobals() -} diff --git a/vendor/github.com/Microsoft/hcsshim/hnsnetwork.go b/vendor/github.com/Microsoft/hcsshim/hnsnetwork.go deleted file mode 100644 index f775fa1d0..000000000 --- a/vendor/github.com/Microsoft/hcsshim/hnsnetwork.go +++ /dev/null @@ -1,36 +0,0 @@ -package hcsshim - -import ( - "github.com/Microsoft/hcsshim/internal/hns" -) - -// Subnet is assoicated with a network and represents a list -// of subnets available to the network -type Subnet = hns.Subnet - -// MacPool is assoicated with a network and represents a list -// of macaddresses available to the network -type MacPool = hns.MacPool - -// HNSNetwork represents a network in HNS -type HNSNetwork = hns.HNSNetwork - -// HNSNetworkRequest makes a call into HNS to update/query a single network -func HNSNetworkRequest(method, path, request string) (*HNSNetwork, error) { - return hns.HNSNetworkRequest(method, path, request) -} - -// HNSListNetworkRequest makes a HNS call to query the list of available networks -func HNSListNetworkRequest(method, path, request string) ([]HNSNetwork, error) { - return hns.HNSListNetworkRequest(method, path, request) -} - -// GetHNSNetworkByID -func GetHNSNetworkByID(networkID string) (*HNSNetwork, error) { - return hns.GetHNSNetworkByID(networkID) -} - -// GetHNSNetworkName filtered by Name -func GetHNSNetworkByName(networkName string) (*HNSNetwork, error) { - return hns.GetHNSNetworkByName(networkName) -} diff --git a/vendor/github.com/Microsoft/hcsshim/hnspolicy.go b/vendor/github.com/Microsoft/hcsshim/hnspolicy.go deleted file mode 100644 index a3e03ff8f..000000000 --- a/vendor/github.com/Microsoft/hcsshim/hnspolicy.go +++ /dev/null @@ -1,57 +0,0 @@ -package hcsshim - -import ( - "github.com/Microsoft/hcsshim/internal/hns" -) - -// Type of Request Support in ModifySystem -type PolicyType = hns.PolicyType - -// RequestType const -const ( - Nat = hns.Nat - ACL = hns.ACL - PA = hns.PA - VLAN = hns.VLAN - VSID = hns.VSID - VNet = hns.VNet - L2Driver = hns.L2Driver - Isolation = hns.Isolation - QOS = hns.QOS - OutboundNat = hns.OutboundNat - ExternalLoadBalancer = hns.ExternalLoadBalancer - Route = hns.Route -) - -type NatPolicy = hns.NatPolicy - -type QosPolicy = hns.QosPolicy - -type IsolationPolicy = hns.IsolationPolicy - -type VlanPolicy = hns.VlanPolicy - -type VsidPolicy = hns.VsidPolicy - -type PaPolicy = hns.PaPolicy - -type OutboundNatPolicy = hns.OutboundNatPolicy - -type ActionType = hns.ActionType -type DirectionType = hns.DirectionType -type RuleType = hns.RuleType - -const ( - Allow = hns.Allow - Block = hns.Block - - In = hns.In - Out = hns.Out - - Host = hns.Host - Switch = hns.Switch -) - -type ACLPolicy = hns.ACLPolicy - -type Policy = hns.Policy diff --git a/vendor/github.com/Microsoft/hcsshim/hnspolicylist.go b/vendor/github.com/Microsoft/hcsshim/hnspolicylist.go deleted file mode 100644 index 55aaa4a50..000000000 --- a/vendor/github.com/Microsoft/hcsshim/hnspolicylist.go +++ /dev/null @@ -1,47 +0,0 @@ -package hcsshim - -import ( - "github.com/Microsoft/hcsshim/internal/hns" -) - -// RoutePolicy is a structure defining schema for Route based Policy -type RoutePolicy = hns.RoutePolicy - -// ELBPolicy is a structure defining schema for ELB LoadBalancing based Policy -type ELBPolicy = hns.ELBPolicy - -// LBPolicy is a structure defining schema for LoadBalancing based Policy -type LBPolicy = hns.LBPolicy - -// PolicyList is a structure defining schema for Policy list request -type PolicyList = hns.PolicyList - -// HNSPolicyListRequest makes a call into HNS to update/query a single network -func HNSPolicyListRequest(method, path, request string) (*PolicyList, error) { - return hns.HNSPolicyListRequest(method, path, request) -} - -// HNSListPolicyListRequest gets all the policy list -func HNSListPolicyListRequest() ([]PolicyList, error) { - return hns.HNSListPolicyListRequest() -} - -// PolicyListRequest makes a HNS call to modify/query a network policy list -func PolicyListRequest(method, path, request string) (*PolicyList, error) { - return hns.PolicyListRequest(method, path, request) -} - -// GetPolicyListByID get the policy list by ID -func GetPolicyListByID(policyListID string) (*PolicyList, error) { - return hns.GetPolicyListByID(policyListID) -} - -// AddLoadBalancer policy list for the specified endpoints -func AddLoadBalancer(endpoints []HNSEndpoint, isILB bool, sourceVIP, vip string, protocol uint16, internalPort uint16, externalPort uint16) (*PolicyList, error) { - return hns.AddLoadBalancer(endpoints, isILB, sourceVIP, vip, protocol, internalPort, externalPort) -} - -// AddRoute adds route policy list for the specified endpoints -func AddRoute(endpoints []HNSEndpoint, destinationPrefix string, nextHop string, encapEnabled bool) (*PolicyList, error) { - return hns.AddRoute(endpoints, destinationPrefix, nextHop, encapEnabled) -} diff --git a/vendor/github.com/Microsoft/hcsshim/hnssupport.go b/vendor/github.com/Microsoft/hcsshim/hnssupport.go deleted file mode 100644 index 69405244b..000000000 --- a/vendor/github.com/Microsoft/hcsshim/hnssupport.go +++ /dev/null @@ -1,13 +0,0 @@ -package hcsshim - -import ( - "github.com/Microsoft/hcsshim/internal/hns" -) - -type HNSSupportedFeatures = hns.HNSSupportedFeatures - -type HNSAclFeatures = hns.HNSAclFeatures - -func GetHNSSupportedFeatures() HNSSupportedFeatures { - return hns.GetHNSSupportedFeatures() -} diff --git a/vendor/github.com/Microsoft/hcsshim/interface.go b/vendor/github.com/Microsoft/hcsshim/interface.go deleted file mode 100644 index 5b91e0cc5..000000000 --- a/vendor/github.com/Microsoft/hcsshim/interface.go +++ /dev/null @@ -1,114 +0,0 @@ -package hcsshim - -import ( - "io" - "time" - - "github.com/Microsoft/hcsshim/internal/schema1" -) - -// ProcessConfig is used as both the input of Container.CreateProcess -// and to convert the parameters to JSON for passing onto the HCS -type ProcessConfig = schema1.ProcessConfig - -type Layer = schema1.Layer -type MappedDir = schema1.MappedDir -type MappedPipe = schema1.MappedPipe -type HvRuntime = schema1.HvRuntime -type MappedVirtualDisk = schema1.MappedVirtualDisk - -// AssignedDevice represents a device that has been directly assigned to a container -// -// NOTE: Support added in RS5 -type AssignedDevice = schema1.AssignedDevice - -// ContainerConfig is used as both the input of CreateContainer -// and to convert the parameters to JSON for passing onto the HCS -type ContainerConfig = schema1.ContainerConfig - -type ComputeSystemQuery = schema1.ComputeSystemQuery - -// Container represents a created (but not necessarily running) container. -type Container interface { - // Start synchronously starts the container. - Start() error - - // Shutdown requests a container shutdown, but it may not actually be shutdown until Wait() succeeds. - Shutdown() error - - // Terminate requests a container terminate, but it may not actually be terminated until Wait() succeeds. - Terminate() error - - // Waits synchronously waits for the container to shutdown or terminate. - Wait() error - - // WaitTimeout synchronously waits for the container to terminate or the duration to elapse. It - // returns false if timeout occurs. - WaitTimeout(time.Duration) error - - // Pause pauses the execution of a container. - Pause() error - - // Resume resumes the execution of a container. - Resume() error - - // HasPendingUpdates returns true if the container has updates pending to install. - HasPendingUpdates() (bool, error) - - // Statistics returns statistics for a container. - Statistics() (Statistics, error) - - // ProcessList returns details for the processes in a container. - ProcessList() ([]ProcessListItem, error) - - // MappedVirtualDisks returns virtual disks mapped to a utility VM, indexed by controller - MappedVirtualDisks() (map[int]MappedVirtualDiskController, error) - - // CreateProcess launches a new process within the container. - CreateProcess(c *ProcessConfig) (Process, error) - - // OpenProcess gets an interface to an existing process within the container. - OpenProcess(pid int) (Process, error) - - // Close cleans up any state associated with the container but does not terminate or wait for it. - Close() error - - // Modify the System - Modify(config *ResourceModificationRequestResponse) error -} - -// Process represents a running or exited process. -type Process interface { - // Pid returns the process ID of the process within the container. - Pid() int - - // Kill signals the process to terminate but does not wait for it to finish terminating. - Kill() error - - // Wait waits for the process to exit. - Wait() error - - // WaitTimeout waits for the process to exit or the duration to elapse. It returns - // false if timeout occurs. - WaitTimeout(time.Duration) error - - // ExitCode returns the exit code of the process. The process must have - // already terminated. - ExitCode() (int, error) - - // ResizeConsole resizes the console of the process. - ResizeConsole(width, height uint16) error - - // Stdio returns the stdin, stdout, and stderr pipes, respectively. Closing - // these pipes does not close the underlying pipes; it should be possible to - // call this multiple times to get multiple interfaces. - Stdio() (io.WriteCloser, io.ReadCloser, io.ReadCloser, error) - - // CloseStdin closes the write side of the stdin pipe so that the process is - // notified on the read side that there is no more data in stdin. - CloseStdin() error - - // Close cleans up any state associated with the process but does not kill - // or wait on it. - Close() error -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/guestrequest/types.go b/vendor/github.com/Microsoft/hcsshim/internal/guestrequest/types.go deleted file mode 100644 index 5d3d0dfef..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/guestrequest/types.go +++ /dev/null @@ -1,100 +0,0 @@ -package guestrequest - -import ( - "github.com/Microsoft/hcsshim/internal/schema2" -) - -// Arguably, many of these (at least CombinedLayers) should have been generated -// by swagger. -// -// This will also change package name due to an inbound breaking change. - -// This class is used by a modify request to add or remove a combined layers -// structure in the guest. For windows, the GCS applies a filter in ContainerRootPath -// using the specified layers as the parent content. Ignores property ScratchPath -// since the container path is already the scratch path. For linux, the GCS unions -// the specified layers and ScratchPath together, placing the resulting union -// filesystem at ContainerRootPath. -type CombinedLayers struct { - ContainerRootPath string `json:"ContainerRootPath,omitempty"` - Layers []hcsschema.Layer `json:"Layers,omitempty"` - ScratchPath string `json:"ScratchPath,omitempty"` -} - -// Defines the schema for hosted settings passed to GCS and/or OpenGCS - -// SCSI. Scratch space for remote file-system commands, or R/W layer for containers -type LCOWMappedVirtualDisk struct { - MountPath string `json:"MountPath,omitempty"` // /tmp/scratch for an LCOW utility VM being used as a service VM - Lun uint8 `json:"Lun,omitempty"` - Controller uint8 `json:"Controller,omitempty"` - ReadOnly bool `json:"ReadOnly,omitempty"` -} - -type WCOWMappedVirtualDisk struct { - ContainerPath string `json:"ContainerPath,omitempty"` - Lun int32 `json:"Lun,omitempty"` -} - -type LCOWMappedDirectory struct { - MountPath string `json:"MountPath,omitempty"` - Port int32 `json:"Port,omitempty"` - ShareName string `json:"ShareName,omitempty"` // If empty not using ANames (not currently supported) - ReadOnly bool `json:"ReadOnly,omitempty"` -} - -// Read-only layers over VPMem -type LCOWMappedVPMemDevice struct { - DeviceNumber uint32 `json:"DeviceNumber,omitempty"` - MountPath string `json:"MountPath,omitempty"` // /tmp/pN -} - -type LCOWNetworkAdapter struct { - NamespaceID string `json:",omitempty"` - ID string `json:",omitempty"` - MacAddress string `json:",omitempty"` - IPAddress string `json:",omitempty"` - PrefixLength uint8 `json:",omitempty"` - GatewayAddress string `json:",omitempty"` - DNSSuffix string `json:",omitempty"` - DNSServerList string `json:",omitempty"` - EnableLowMetric bool `json:",omitempty"` - EncapOverhead uint16 `json:",omitempty"` -} - -type ResourceType string - -const ( - // These are constants for v2 schema modify guest requests. - ResourceTypeMappedDirectory ResourceType = "MappedDirectory" - ResourceTypeMappedVirtualDisk ResourceType = "MappedVirtualDisk" - ResourceTypeNetwork ResourceType = "Network" - ResourceTypeNetworkNamespace ResourceType = "NetworkNamespace" - ResourceTypeCombinedLayers ResourceType = "CombinedLayers" - ResourceTypeVPMemDevice ResourceType = "VPMemDevice" -) - -// GuestRequest is for modify commands passed to the guest. -type GuestRequest struct { - RequestType string `json:"RequestType,omitempty"` - ResourceType ResourceType `json:"ResourceType,omitempty"` - Settings interface{} `json:"Settings,omitempty"` -} - -type NetworkModifyRequest struct { - AdapterId string `json:"AdapterId,omitempty"` - RequestType string `json:"RequestType,omitempty"` - Settings interface{} `json:"Settings,omitempty"` -} - -type RS4NetworkModifyRequest struct { - AdapterInstanceId string `json:"AdapterInstanceId,omitempty"` - RequestType string `json:"RequestType,omitempty"` - Settings interface{} `json:"Settings,omitempty"` -} - -// SignalProcessOptions is the options passed to either WCOW or LCOW -// to signal a given process. -type SignalProcessOptions struct { - Signal int `json:,omitempty` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/guid/guid.go b/vendor/github.com/Microsoft/hcsshim/internal/guid/guid.go deleted file mode 100644 index e9e45c030..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/guid/guid.go +++ /dev/null @@ -1,69 +0,0 @@ -package guid - -import ( - "crypto/rand" - "encoding/json" - "fmt" - "io" - "strconv" - "strings" -) - -var _ = (json.Marshaler)(&GUID{}) -var _ = (json.Unmarshaler)(&GUID{}) - -type GUID [16]byte - -func New() GUID { - g := GUID{} - _, err := io.ReadFull(rand.Reader, g[:]) - if err != nil { - panic(err) - } - return g -} - -func (g GUID) String() string { - return fmt.Sprintf("%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x-%02x", g[3], g[2], g[1], g[0], g[5], g[4], g[7], g[6], g[8:10], g[10:]) -} - -func FromString(s string) GUID { - if len(s) != 36 { - panic(fmt.Sprintf("invalid GUID length: %d", len(s))) - } - if s[8] != '-' || s[13] != '-' || s[18] != '-' || s[23] != '-' { - panic("invalid GUID format") - } - indexOrder := [16]int{ - 0, 2, 4, 6, - 9, 11, - 14, 16, - 19, 21, - 24, 26, 28, 30, 32, 34, - } - byteOrder := [16]int{ - 3, 2, 1, 0, - 5, 4, - 7, 6, - 8, 9, - 10, 11, 12, 13, 14, 15, - } - var g GUID - for i, x := range indexOrder { - b, err := strconv.ParseInt(s[x:x+2], 16, 16) - if err != nil { - panic(err) - } - g[byteOrder[i]] = byte(b) - } - return g -} - -func (g GUID) MarshalJSON() ([]byte, error) { - return json.Marshal(g.String()) -} - -func (g *GUID) UnmarshalJSON(data []byte) error { - *g = FromString(strings.Trim(string(data), "\"")) - return nil -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hcs/callback.go b/vendor/github.com/Microsoft/hcsshim/internal/hcs/callback.go deleted file mode 100644 index f9a922a4b..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/hcs/callback.go +++ /dev/null @@ -1,104 +0,0 @@ -package hcs - -import ( - "sync" - "syscall" - - "github.com/Microsoft/hcsshim/internal/interop" - "github.com/sirupsen/logrus" -) - -var ( - nextCallback uintptr - callbackMap = map[uintptr]*notifcationWatcherContext{} - callbackMapLock = sync.RWMutex{} - - notificationWatcherCallback = syscall.NewCallback(notificationWatcher) - - // Notifications for HCS_SYSTEM handles - hcsNotificationSystemExited hcsNotification = 0x00000001 - hcsNotificationSystemCreateCompleted hcsNotification = 0x00000002 - hcsNotificationSystemStartCompleted hcsNotification = 0x00000003 - hcsNotificationSystemPauseCompleted hcsNotification = 0x00000004 - hcsNotificationSystemResumeCompleted hcsNotification = 0x00000005 - hcsNotificationSystemCrashReport hcsNotification = 0x00000006 - hcsNotificationSystemSiloJobCreated hcsNotification = 0x00000007 - hcsNotificationSystemSaveCompleted hcsNotification = 0x00000008 - hcsNotificationSystemRdpEnhancedModeStateChanged hcsNotification = 0x00000009 - hcsNotificationSystemShutdownFailed hcsNotification = 0x0000000A - hcsNotificationSystemGetPropertiesCompleted hcsNotification = 0x0000000B - hcsNotificationSystemModifyCompleted hcsNotification = 0x0000000C - hcsNotificationSystemCrashInitiated hcsNotification = 0x0000000D - hcsNotificationSystemGuestConnectionClosed hcsNotification = 0x0000000E - - // Notifications for HCS_PROCESS handles - hcsNotificationProcessExited hcsNotification = 0x00010000 - - // Common notifications - hcsNotificationInvalid hcsNotification = 0x00000000 - hcsNotificationServiceDisconnect hcsNotification = 0x01000000 -) - -type hcsNotification uint32 -type notificationChannel chan error - -type notifcationWatcherContext struct { - channels notificationChannels - handle hcsCallback -} - -type notificationChannels map[hcsNotification]notificationChannel - -func newChannels() notificationChannels { - channels := make(notificationChannels) - - channels[hcsNotificationSystemExited] = make(notificationChannel, 1) - channels[hcsNotificationSystemCreateCompleted] = make(notificationChannel, 1) - channels[hcsNotificationSystemStartCompleted] = make(notificationChannel, 1) - channels[hcsNotificationSystemPauseCompleted] = make(notificationChannel, 1) - channels[hcsNotificationSystemResumeCompleted] = make(notificationChannel, 1) - channels[hcsNotificationProcessExited] = make(notificationChannel, 1) - channels[hcsNotificationServiceDisconnect] = make(notificationChannel, 1) - channels[hcsNotificationSystemCrashReport] = make(notificationChannel, 1) - channels[hcsNotificationSystemSiloJobCreated] = make(notificationChannel, 1) - channels[hcsNotificationSystemSaveCompleted] = make(notificationChannel, 1) - channels[hcsNotificationSystemRdpEnhancedModeStateChanged] = make(notificationChannel, 1) - channels[hcsNotificationSystemShutdownFailed] = make(notificationChannel, 1) - channels[hcsNotificationSystemGetPropertiesCompleted] = make(notificationChannel, 1) - channels[hcsNotificationSystemModifyCompleted] = make(notificationChannel, 1) - channels[hcsNotificationSystemCrashInitiated] = make(notificationChannel, 1) - channels[hcsNotificationSystemGuestConnectionClosed] = make(notificationChannel, 1) - - return channels -} - -func closeChannels(channels notificationChannels) { - for _, c := range channels { - close(c) - } -} - -func notificationWatcher(notificationType hcsNotification, callbackNumber uintptr, notificationStatus uintptr, notificationData *uint16) uintptr { - var result error - if int32(notificationStatus) < 0 { - result = interop.Win32FromHresult(notificationStatus) - } - - callbackMapLock.RLock() - context := callbackMap[callbackNumber] - callbackMapLock.RUnlock() - - if context == nil { - return 0 - } - - if channel, ok := context.channels[notificationType]; ok { - channel <- result - } else { - logrus.WithFields(logrus.Fields{ - "notification-type": notificationType, - }).Warn("Received a callback of an unsupported type") - } - - return 0 -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hcs/cgo.go b/vendor/github.com/Microsoft/hcsshim/internal/hcs/cgo.go deleted file mode 100644 index 3669c34aa..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/hcs/cgo.go +++ /dev/null @@ -1,7 +0,0 @@ -package hcs - -import "C" - -// This import is needed to make the library compile as CGO because HCSSHIM -// only works with CGO due to callbacks from HCS comming back from a C thread -// which is not supported without CGO. See https://github.com/golang/go/issues/10973 diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hcs/errors.go b/vendor/github.com/Microsoft/hcsshim/internal/hcs/errors.go deleted file mode 100644 index 079b56535..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/hcs/errors.go +++ /dev/null @@ -1,287 +0,0 @@ -package hcs - -import ( - "encoding/json" - "errors" - "fmt" - "syscall" - - "github.com/Microsoft/hcsshim/internal/interop" - "github.com/Microsoft/hcsshim/internal/logfields" - "github.com/sirupsen/logrus" -) - -var ( - // ErrComputeSystemDoesNotExist is an error encountered when the container being operated on no longer exists - ErrComputeSystemDoesNotExist = syscall.Errno(0xc037010e) - - // ErrElementNotFound is an error encountered when the object being referenced does not exist - ErrElementNotFound = syscall.Errno(0x490) - - // ErrElementNotFound is an error encountered when the object being referenced does not exist - ErrNotSupported = syscall.Errno(0x32) - - // ErrInvalidData is an error encountered when the request being sent to hcs is invalid/unsupported - // decimal -2147024883 / hex 0x8007000d - ErrInvalidData = syscall.Errno(0xd) - - // ErrHandleClose is an error encountered when the handle generating the notification being waited on has been closed - ErrHandleClose = errors.New("hcsshim: the handle generating this notification has been closed") - - // ErrAlreadyClosed is an error encountered when using a handle that has been closed by the Close method - ErrAlreadyClosed = errors.New("hcsshim: the handle has already been closed") - - // ErrInvalidNotificationType is an error encountered when an invalid notification type is used - ErrInvalidNotificationType = errors.New("hcsshim: invalid notification type") - - // ErrInvalidProcessState is an error encountered when the process is not in a valid state for the requested operation - ErrInvalidProcessState = errors.New("the process is in an invalid state for the attempted operation") - - // ErrTimeout is an error encountered when waiting on a notification times out - ErrTimeout = errors.New("hcsshim: timeout waiting for notification") - - // ErrUnexpectedContainerExit is the error encountered when a container exits while waiting for - // a different expected notification - ErrUnexpectedContainerExit = errors.New("unexpected container exit") - - // ErrUnexpectedProcessAbort is the error encountered when communication with the compute service - // is lost while waiting for a notification - ErrUnexpectedProcessAbort = errors.New("lost communication with compute service") - - // ErrUnexpectedValue is an error encountered when hcs returns an invalid value - ErrUnexpectedValue = errors.New("unexpected value returned from hcs") - - // ErrVmcomputeAlreadyStopped is an error encountered when a shutdown or terminate request is made on a stopped container - ErrVmcomputeAlreadyStopped = syscall.Errno(0xc0370110) - - // ErrVmcomputeOperationPending is an error encountered when the operation is being completed asynchronously - ErrVmcomputeOperationPending = syscall.Errno(0xC0370103) - - // ErrVmcomputeOperationInvalidState is an error encountered when the compute system is not in a valid state for the requested operation - ErrVmcomputeOperationInvalidState = syscall.Errno(0xc0370105) - - // ErrProcNotFound is an error encountered when the the process cannot be found - ErrProcNotFound = syscall.Errno(0x7f) - - // ErrVmcomputeOperationAccessIsDenied is an error which can be encountered when enumerating compute systems in RS1/RS2 - // builds when the underlying silo might be in the process of terminating. HCS was fixed in RS3. - ErrVmcomputeOperationAccessIsDenied = syscall.Errno(0x5) - - // ErrVmcomputeInvalidJSON is an error encountered when the compute system does not support/understand the messages sent by management - ErrVmcomputeInvalidJSON = syscall.Errno(0xc037010d) - - // ErrVmcomputeUnknownMessage is an error encountered guest compute system doesn't support the message - ErrVmcomputeUnknownMessage = syscall.Errno(0xc037010b) - - // ErrVmcomputeUnexpectedExit is an error encountered when the compute system terminates unexpectedly - ErrVmcomputeUnexpectedExit = syscall.Errno(0xC0370106) - - // ErrNotSupported is an error encountered when hcs doesn't support the request - ErrPlatformNotSupported = errors.New("unsupported platform request") -) - -type ErrorEvent struct { - Message string `json:"Message,omitempty"` // Fully formated error message - StackTrace string `json:"StackTrace,omitempty"` // Stack trace in string form - Provider string `json:"Provider,omitempty"` - EventID uint16 `json:"EventId,omitempty"` - Flags uint32 `json:"Flags,omitempty"` - Source string `json:"Source,omitempty"` - //Data []EventData `json:"Data,omitempty"` // Omit this as HCS doesn't encode this well. It's more confusing to include. It is however logged in debug mode (see processHcsResult function) -} - -type hcsResult struct { - Error int32 - ErrorMessage string - ErrorEvents []ErrorEvent `json:"ErrorEvents,omitempty"` -} - -func (ev *ErrorEvent) String() string { - evs := "[Event Detail: " + ev.Message - if ev.StackTrace != "" { - evs += " Stack Trace: " + ev.StackTrace - } - if ev.Provider != "" { - evs += " Provider: " + ev.Provider - } - if ev.EventID != 0 { - evs = fmt.Sprintf("%s EventID: %d", evs, ev.EventID) - } - if ev.Flags != 0 { - evs = fmt.Sprintf("%s flags: %d", evs, ev.Flags) - } - if ev.Source != "" { - evs += " Source: " + ev.Source - } - evs += "]" - return evs -} - -func processHcsResult(resultp *uint16) []ErrorEvent { - if resultp != nil { - resultj := interop.ConvertAndFreeCoTaskMemString(resultp) - logrus.WithField(logfields.JSON, resultj). - Debug("HCS Result") - result := &hcsResult{} - if err := json.Unmarshal([]byte(resultj), result); err != nil { - logrus.WithFields(logrus.Fields{ - logfields.JSON: resultj, - logrus.ErrorKey: err, - }).Warning("Could not unmarshal HCS result") - return nil - } - return result.ErrorEvents - } - return nil -} - -type HcsError struct { - Op string - Err error - Events []ErrorEvent -} - -func (e *HcsError) Error() string { - s := e.Op + ": " + e.Err.Error() - for _, ev := range e.Events { - s += "\n" + ev.String() - } - return s -} - -// ProcessError is an error encountered in HCS during an operation on a Process object -type ProcessError struct { - SystemID string - Pid int - Op string - Err error - Events []ErrorEvent -} - -// SystemError is an error encountered in HCS during an operation on a Container object -type SystemError struct { - ID string - Op string - Err error - Extra string - Events []ErrorEvent -} - -func (e *SystemError) Error() string { - s := e.Op + " " + e.ID + ": " + e.Err.Error() - for _, ev := range e.Events { - s += "\n" + ev.String() - } - if e.Extra != "" { - s += "\n(extra info: " + e.Extra + ")" - } - return s -} - -func makeSystemError(system *System, op string, extra string, err error, events []ErrorEvent) error { - // Don't double wrap errors - if _, ok := err.(*SystemError); ok { - return err - } - return &SystemError{ - ID: system.ID(), - Op: op, - Extra: extra, - Err: err, - Events: events, - } -} - -func (e *ProcessError) Error() string { - s := fmt.Sprintf("%s %s:%d: %s", e.Op, e.SystemID, e.Pid, e.Err.Error()) - for _, ev := range e.Events { - s += "\n" + ev.String() - } - return s -} - -func makeProcessError(process *Process, op string, err error, events []ErrorEvent) error { - // Don't double wrap errors - if _, ok := err.(*ProcessError); ok { - return err - } - return &ProcessError{ - Pid: process.Pid(), - SystemID: process.SystemID(), - Op: op, - Err: err, - Events: events, - } -} - -// IsNotExist checks if an error is caused by the Container or Process not existing. -// Note: Currently, ErrElementNotFound can mean that a Process has either -// already exited, or does not exist. Both IsAlreadyStopped and IsNotExist -// will currently return true when the error is ErrElementNotFound or ErrProcNotFound. -func IsNotExist(err error) bool { - err = getInnerError(err) - return err == ErrComputeSystemDoesNotExist || - err == ErrElementNotFound || - err == ErrProcNotFound -} - -// IsAlreadyClosed checks if an error is caused by the Container or Process having been -// already closed by a call to the Close() method. -func IsAlreadyClosed(err error) bool { - err = getInnerError(err) - return err == ErrAlreadyClosed -} - -// IsPending returns a boolean indicating whether the error is that -// the requested operation is being completed in the background. -func IsPending(err error) bool { - err = getInnerError(err) - return err == ErrVmcomputeOperationPending -} - -// IsTimeout returns a boolean indicating whether the error is caused by -// a timeout waiting for the operation to complete. -func IsTimeout(err error) bool { - err = getInnerError(err) - return err == ErrTimeout -} - -// IsAlreadyStopped returns a boolean indicating whether the error is caused by -// a Container or Process being already stopped. -// Note: Currently, ErrElementNotFound can mean that a Process has either -// already exited, or does not exist. Both IsAlreadyStopped and IsNotExist -// will currently return true when the error is ErrElementNotFound or ErrProcNotFound. -func IsAlreadyStopped(err error) bool { - err = getInnerError(err) - return err == ErrVmcomputeAlreadyStopped || - err == ErrElementNotFound || - err == ErrProcNotFound -} - -// IsNotSupported returns a boolean indicating whether the error is caused by -// unsupported platform requests -// Note: Currently Unsupported platform requests can be mean either -// ErrVmcomputeInvalidJSON, ErrInvalidData, ErrNotSupported or ErrVmcomputeUnknownMessage -// is thrown from the Platform -func IsNotSupported(err error) bool { - err = getInnerError(err) - // If Platform doesn't recognize or support the request sent, below errors are seen - return err == ErrVmcomputeInvalidJSON || - err == ErrInvalidData || - err == ErrNotSupported || - err == ErrVmcomputeUnknownMessage -} - -func getInnerError(err error) error { - switch pe := err.(type) { - case nil: - return nil - case *HcsError: - err = pe.Err - case *SystemError: - err = pe.Err - case *ProcessError: - err = pe.Err - } - return err -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hcs/hcs.go b/vendor/github.com/Microsoft/hcsshim/internal/hcs/hcs.go deleted file mode 100644 index b0d49cbcf..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/hcs/hcs.go +++ /dev/null @@ -1,48 +0,0 @@ -// Shim for the Host Compute Service (HCS) to manage Windows Server -// containers and Hyper-V containers. - -package hcs - -import ( - "syscall" -) - -//go:generate go run ../../mksyscall_windows.go -output zsyscall_windows.go hcs.go - -//sys hcsEnumerateComputeSystems(query string, computeSystems **uint16, result **uint16) (hr error) = vmcompute.HcsEnumerateComputeSystems? -//sys hcsCreateComputeSystem(id string, configuration string, identity syscall.Handle, computeSystem *hcsSystem, result **uint16) (hr error) = vmcompute.HcsCreateComputeSystem? -//sys hcsOpenComputeSystem(id string, computeSystem *hcsSystem, result **uint16) (hr error) = vmcompute.HcsOpenComputeSystem? -//sys hcsCloseComputeSystem(computeSystem hcsSystem) (hr error) = vmcompute.HcsCloseComputeSystem? -//sys hcsStartComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) = vmcompute.HcsStartComputeSystem? -//sys hcsShutdownComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) = vmcompute.HcsShutdownComputeSystem? -//sys hcsTerminateComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) = vmcompute.HcsTerminateComputeSystem? -//sys hcsPauseComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) = vmcompute.HcsPauseComputeSystem? -//sys hcsResumeComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) = vmcompute.HcsResumeComputeSystem? -//sys hcsGetComputeSystemProperties(computeSystem hcsSystem, propertyQuery string, properties **uint16, result **uint16) (hr error) = vmcompute.HcsGetComputeSystemProperties? -//sys hcsModifyComputeSystem(computeSystem hcsSystem, configuration string, result **uint16) (hr error) = vmcompute.HcsModifyComputeSystem? -//sys hcsRegisterComputeSystemCallback(computeSystem hcsSystem, callback uintptr, context uintptr, callbackHandle *hcsCallback) (hr error) = vmcompute.HcsRegisterComputeSystemCallback? -//sys hcsUnregisterComputeSystemCallback(callbackHandle hcsCallback) (hr error) = vmcompute.HcsUnregisterComputeSystemCallback? - -//sys hcsCreateProcess(computeSystem hcsSystem, processParameters string, processInformation *hcsProcessInformation, process *hcsProcess, result **uint16) (hr error) = vmcompute.HcsCreateProcess? -//sys hcsOpenProcess(computeSystem hcsSystem, pid uint32, process *hcsProcess, result **uint16) (hr error) = vmcompute.HcsOpenProcess? -//sys hcsCloseProcess(process hcsProcess) (hr error) = vmcompute.HcsCloseProcess? -//sys hcsTerminateProcess(process hcsProcess, result **uint16) (hr error) = vmcompute.HcsTerminateProcess? -//sys hcsSignalProcess(process hcsProcess, options string, result **uint16) (hr error) = vmcompute.HcsTerminateProcess? -//sys hcsGetProcessInfo(process hcsProcess, processInformation *hcsProcessInformation, result **uint16) (hr error) = vmcompute.HcsGetProcessInfo? -//sys hcsGetProcessProperties(process hcsProcess, processProperties **uint16, result **uint16) (hr error) = vmcompute.HcsGetProcessProperties? -//sys hcsModifyProcess(process hcsProcess, settings string, result **uint16) (hr error) = vmcompute.HcsModifyProcess? -//sys hcsGetServiceProperties(propertyQuery string, properties **uint16, result **uint16) (hr error) = vmcompute.HcsGetServiceProperties? -//sys hcsRegisterProcessCallback(process hcsProcess, callback uintptr, context uintptr, callbackHandle *hcsCallback) (hr error) = vmcompute.HcsRegisterProcessCallback? -//sys hcsUnregisterProcessCallback(callbackHandle hcsCallback) (hr error) = vmcompute.HcsUnregisterProcessCallback? - -type hcsSystem syscall.Handle -type hcsProcess syscall.Handle -type hcsCallback syscall.Handle - -type hcsProcessInformation struct { - ProcessId uint32 - Reserved uint32 - StdInput syscall.Handle - StdOutput syscall.Handle - StdError syscall.Handle -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hcs/log.go b/vendor/github.com/Microsoft/hcsshim/internal/hcs/log.go deleted file mode 100644 index 6d03b17a2..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/hcs/log.go +++ /dev/null @@ -1,20 +0,0 @@ -package hcs - -import "github.com/sirupsen/logrus" - -func logOperationBegin(ctx logrus.Fields, msg string) { - logrus.WithFields(ctx).Debug(msg) -} - -func logOperationEnd(ctx logrus.Fields, msg string, err error) { - // Copy the log and fields first. - log := logrus.WithFields(ctx) - if err == nil { - log.Debug(msg) - } else { - // Edit only the copied field data to avoid race conditions on the - // write. - log.Data[logrus.ErrorKey] = err - log.Error(msg) - } -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hcs/process.go b/vendor/github.com/Microsoft/hcsshim/internal/hcs/process.go deleted file mode 100644 index 41e20bbf9..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/hcs/process.go +++ /dev/null @@ -1,459 +0,0 @@ -package hcs - -import ( - "encoding/json" - "io" - "sync" - "syscall" - "time" - - "github.com/Microsoft/hcsshim/internal/guestrequest" - "github.com/Microsoft/hcsshim/internal/interop" - "github.com/Microsoft/hcsshim/internal/logfields" - "github.com/sirupsen/logrus" -) - -// ContainerError is an error encountered in HCS -type Process struct { - handleLock sync.RWMutex - handle hcsProcess - processID int - system *System - cachedPipes *cachedPipes - callbackNumber uintptr - - logctx logrus.Fields -} - -func newProcess(process hcsProcess, processID int, computeSystem *System) *Process { - return &Process{ - handle: process, - processID: processID, - system: computeSystem, - logctx: logrus.Fields{ - logfields.ContainerID: computeSystem.ID(), - logfields.ProcessID: processID, - }, - } -} - -type cachedPipes struct { - stdIn syscall.Handle - stdOut syscall.Handle - stdErr syscall.Handle -} - -type processModifyRequest struct { - Operation string - ConsoleSize *consoleSize `json:",omitempty"` - CloseHandle *closeHandle `json:",omitempty"` -} - -type consoleSize struct { - Height uint16 - Width uint16 -} - -type closeHandle struct { - Handle string -} - -type ProcessStatus struct { - ProcessID uint32 - Exited bool - ExitCode uint32 - LastWaitResult int32 -} - -const ( - stdIn string = "StdIn" - stdOut string = "StdOut" - stdErr string = "StdErr" -) - -const ( - modifyConsoleSize string = "ConsoleSize" - modifyCloseHandle string = "CloseHandle" -) - -// Pid returns the process ID of the process within the container. -func (process *Process) Pid() int { - return process.processID -} - -// SystemID returns the ID of the process's compute system. -func (process *Process) SystemID() string { - return process.system.ID() -} - -func (process *Process) logOperationBegin(operation string) { - logOperationBegin( - process.logctx, - operation+" - Begin Operation") -} - -func (process *Process) logOperationEnd(operation string, err error) { - var result string - if err == nil { - result = "Success" - } else { - result = "Error" - } - - logOperationEnd( - process.logctx, - operation+" - End Operation - "+result, - err) -} - -// Signal signals the process with `options`. -func (process *Process) Signal(options guestrequest.SignalProcessOptions) (err error) { - process.handleLock.RLock() - defer process.handleLock.RUnlock() - - operation := "hcsshim::Process::Signal" - process.logOperationBegin(operation) - defer func() { process.logOperationEnd(operation, err) }() - - if process.handle == 0 { - return makeProcessError(process, operation, ErrAlreadyClosed, nil) - } - - optionsb, err := json.Marshal(options) - if err != nil { - return err - } - - optionsStr := string(optionsb) - - var resultp *uint16 - syscallWatcher(process.logctx, func() { - err = hcsSignalProcess(process.handle, optionsStr, &resultp) - }) - events := processHcsResult(resultp) - if err != nil { - return makeProcessError(process, operation, err, events) - } - - return nil -} - -// Kill signals the process to terminate but does not wait for it to finish terminating. -func (process *Process) Kill() (err error) { - process.handleLock.RLock() - defer process.handleLock.RUnlock() - - operation := "hcsshim::Process::Kill" - process.logOperationBegin(operation) - defer func() { process.logOperationEnd(operation, err) }() - - if process.handle == 0 { - return makeProcessError(process, operation, ErrAlreadyClosed, nil) - } - - var resultp *uint16 - syscallWatcher(process.logctx, func() { - err = hcsTerminateProcess(process.handle, &resultp) - }) - events := processHcsResult(resultp) - if err != nil { - return makeProcessError(process, operation, err, events) - } - - return nil -} - -// Wait waits for the process to exit. -func (process *Process) Wait() (err error) { - operation := "hcsshim::Process::Wait" - process.logOperationBegin(operation) - defer func() { process.logOperationEnd(operation, err) }() - - err = waitForNotification(process.callbackNumber, hcsNotificationProcessExited, nil) - if err != nil { - return makeProcessError(process, operation, err, nil) - } - - return nil -} - -// WaitTimeout waits for the process to exit or the duration to elapse. It returns -// false if timeout occurs. -func (process *Process) WaitTimeout(timeout time.Duration) (err error) { - operation := "hcssshim::Process::WaitTimeout" - process.logOperationBegin(operation) - defer func() { process.logOperationEnd(operation, err) }() - - err = waitForNotification(process.callbackNumber, hcsNotificationProcessExited, &timeout) - if err != nil { - return makeProcessError(process, operation, err, nil) - } - - return nil -} - -// ResizeConsole resizes the console of the process. -func (process *Process) ResizeConsole(width, height uint16) (err error) { - process.handleLock.RLock() - defer process.handleLock.RUnlock() - - operation := "hcsshim::Process::ResizeConsole" - process.logOperationBegin(operation) - defer func() { process.logOperationEnd(operation, err) }() - - if process.handle == 0 { - return makeProcessError(process, operation, ErrAlreadyClosed, nil) - } - - modifyRequest := processModifyRequest{ - Operation: modifyConsoleSize, - ConsoleSize: &consoleSize{ - Height: height, - Width: width, - }, - } - - modifyRequestb, err := json.Marshal(modifyRequest) - if err != nil { - return err - } - - modifyRequestStr := string(modifyRequestb) - - var resultp *uint16 - err = hcsModifyProcess(process.handle, modifyRequestStr, &resultp) - events := processHcsResult(resultp) - if err != nil { - return makeProcessError(process, operation, err, events) - } - - return nil -} - -func (process *Process) Properties() (_ *ProcessStatus, err error) { - process.handleLock.RLock() - defer process.handleLock.RUnlock() - - operation := "hcsshim::Process::Properties" - process.logOperationBegin(operation) - defer func() { process.logOperationEnd(operation, err) }() - - if process.handle == 0 { - return nil, makeProcessError(process, operation, ErrAlreadyClosed, nil) - } - - var ( - resultp *uint16 - propertiesp *uint16 - ) - syscallWatcher(process.logctx, func() { - err = hcsGetProcessProperties(process.handle, &propertiesp, &resultp) - }) - events := processHcsResult(resultp) - if err != nil { - return nil, makeProcessError(process, operation, err, events) - } - - if propertiesp == nil { - return nil, ErrUnexpectedValue - } - propertiesRaw := interop.ConvertAndFreeCoTaskMemBytes(propertiesp) - - properties := &ProcessStatus{} - if err := json.Unmarshal(propertiesRaw, properties); err != nil { - return nil, makeProcessError(process, operation, err, nil) - } - - return properties, nil -} - -// ExitCode returns the exit code of the process. The process must have -// already terminated. -func (process *Process) ExitCode() (_ int, err error) { - operation := "hcsshim::Process::ExitCode" - process.logOperationBegin(operation) - defer func() { process.logOperationEnd(operation, err) }() - - properties, err := process.Properties() - if err != nil { - return 0, makeProcessError(process, operation, err, nil) - } - - if properties.Exited == false { - return 0, makeProcessError(process, operation, ErrInvalidProcessState, nil) - } - - if properties.LastWaitResult != 0 { - return 0, makeProcessError(process, operation, syscall.Errno(properties.LastWaitResult), nil) - } - - return int(properties.ExitCode), nil -} - -// Stdio returns the stdin, stdout, and stderr pipes, respectively. Closing -// these pipes does not close the underlying pipes; it should be possible to -// call this multiple times to get multiple interfaces. -func (process *Process) Stdio() (_ io.WriteCloser, _ io.ReadCloser, _ io.ReadCloser, err error) { - process.handleLock.RLock() - defer process.handleLock.RUnlock() - - operation := "hcsshim::Process::Stdio" - process.logOperationBegin(operation) - defer func() { process.logOperationEnd(operation, err) }() - - if process.handle == 0 { - return nil, nil, nil, makeProcessError(process, operation, ErrAlreadyClosed, nil) - } - - var stdIn, stdOut, stdErr syscall.Handle - - if process.cachedPipes == nil { - var ( - processInfo hcsProcessInformation - resultp *uint16 - ) - err = hcsGetProcessInfo(process.handle, &processInfo, &resultp) - events := processHcsResult(resultp) - if err != nil { - return nil, nil, nil, makeProcessError(process, operation, err, events) - } - - stdIn, stdOut, stdErr = processInfo.StdInput, processInfo.StdOutput, processInfo.StdError - } else { - // Use cached pipes - stdIn, stdOut, stdErr = process.cachedPipes.stdIn, process.cachedPipes.stdOut, process.cachedPipes.stdErr - - // Invalidate the cache - process.cachedPipes = nil - } - - pipes, err := makeOpenFiles([]syscall.Handle{stdIn, stdOut, stdErr}) - if err != nil { - return nil, nil, nil, makeProcessError(process, operation, err, nil) - } - - return pipes[0], pipes[1], pipes[2], nil -} - -// CloseStdin closes the write side of the stdin pipe so that the process is -// notified on the read side that there is no more data in stdin. -func (process *Process) CloseStdin() (err error) { - process.handleLock.RLock() - defer process.handleLock.RUnlock() - - operation := "hcsshim::Process::CloseStdin" - process.logOperationBegin(operation) - defer func() { process.logOperationEnd(operation, err) }() - - if process.handle == 0 { - return makeProcessError(process, operation, ErrAlreadyClosed, nil) - } - - modifyRequest := processModifyRequest{ - Operation: modifyCloseHandle, - CloseHandle: &closeHandle{ - Handle: stdIn, - }, - } - - modifyRequestb, err := json.Marshal(modifyRequest) - if err != nil { - return err - } - - modifyRequestStr := string(modifyRequestb) - - var resultp *uint16 - err = hcsModifyProcess(process.handle, modifyRequestStr, &resultp) - events := processHcsResult(resultp) - if err != nil { - return makeProcessError(process, operation, err, events) - } - - return nil -} - -// Close cleans up any state associated with the process but does not kill -// or wait on it. -func (process *Process) Close() (err error) { - process.handleLock.Lock() - defer process.handleLock.Unlock() - - operation := "hcsshim::Process::Close" - process.logOperationBegin(operation) - defer func() { process.logOperationEnd(operation, err) }() - - // Don't double free this - if process.handle == 0 { - return nil - } - - if err = process.unregisterCallback(); err != nil { - return makeProcessError(process, operation, err, nil) - } - - if err = hcsCloseProcess(process.handle); err != nil { - return makeProcessError(process, operation, err, nil) - } - - process.handle = 0 - - return nil -} - -func (process *Process) registerCallback() error { - context := ¬ifcationWatcherContext{ - channels: newChannels(), - } - - callbackMapLock.Lock() - callbackNumber := nextCallback - nextCallback++ - callbackMap[callbackNumber] = context - callbackMapLock.Unlock() - - var callbackHandle hcsCallback - err := hcsRegisterProcessCallback(process.handle, notificationWatcherCallback, callbackNumber, &callbackHandle) - if err != nil { - return err - } - context.handle = callbackHandle - process.callbackNumber = callbackNumber - - return nil -} - -func (process *Process) unregisterCallback() error { - callbackNumber := process.callbackNumber - - callbackMapLock.RLock() - context := callbackMap[callbackNumber] - callbackMapLock.RUnlock() - - if context == nil { - return nil - } - - handle := context.handle - - if handle == 0 { - return nil - } - - // hcsUnregisterProcessCallback has its own syncronization - // to wait for all callbacks to complete. We must NOT hold the callbackMapLock. - err := hcsUnregisterProcessCallback(handle) - if err != nil { - return err - } - - closeChannels(context.channels) - - callbackMapLock.Lock() - callbackMap[callbackNumber] = nil - callbackMapLock.Unlock() - - handle = 0 - - return nil -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hcs/system.go b/vendor/github.com/Microsoft/hcsshim/internal/hcs/system.go deleted file mode 100644 index 20b242524..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/hcs/system.go +++ /dev/null @@ -1,685 +0,0 @@ -package hcs - -import ( - "encoding/json" - "os" - "strconv" - "sync" - "syscall" - "time" - - "github.com/Microsoft/hcsshim/internal/interop" - "github.com/Microsoft/hcsshim/internal/logfields" - "github.com/Microsoft/hcsshim/internal/schema1" - "github.com/Microsoft/hcsshim/internal/timeout" - "github.com/sirupsen/logrus" -) - -// currentContainerStarts is used to limit the number of concurrent container -// starts. -var currentContainerStarts containerStarts - -type containerStarts struct { - maxParallel int - inProgress int - sync.Mutex -} - -func init() { - mpsS := os.Getenv("HCSSHIM_MAX_PARALLEL_START") - if len(mpsS) > 0 { - mpsI, err := strconv.Atoi(mpsS) - if err != nil || mpsI < 0 { - return - } - currentContainerStarts.maxParallel = mpsI - } -} - -type System struct { - handleLock sync.RWMutex - handle hcsSystem - id string - callbackNumber uintptr - - logctx logrus.Fields -} - -func newSystem(id string) *System { - return &System{ - id: id, - logctx: logrus.Fields{ - logfields.ContainerID: id, - }, - } -} - -func (computeSystem *System) logOperationBegin(operation string) { - logOperationBegin( - computeSystem.logctx, - operation+" - Begin Operation") -} - -func (computeSystem *System) logOperationEnd(operation string, err error) { - var result string - if err == nil { - result = "Success" - } else { - result = "Error" - } - - logOperationEnd( - computeSystem.logctx, - operation+" - End Operation - "+result, - err) -} - -// CreateComputeSystem creates a new compute system with the given configuration but does not start it. -func CreateComputeSystem(id string, hcsDocumentInterface interface{}) (_ *System, err error) { - operation := "hcsshim::CreateComputeSystem" - - computeSystem := newSystem(id) - computeSystem.logOperationBegin(operation) - defer func() { computeSystem.logOperationEnd(operation, err) }() - - hcsDocumentB, err := json.Marshal(hcsDocumentInterface) - if err != nil { - return nil, err - } - - hcsDocument := string(hcsDocumentB) - - logrus.WithFields(computeSystem.logctx). - WithField(logfields.JSON, hcsDocument). - Debug("HCS ComputeSystem Document") - - var ( - resultp *uint16 - identity syscall.Handle - createError error - ) - syscallWatcher(computeSystem.logctx, func() { - createError = hcsCreateComputeSystem(id, hcsDocument, identity, &computeSystem.handle, &resultp) - }) - - if createError == nil || IsPending(createError) { - if err = computeSystem.registerCallback(); err != nil { - // Terminate the compute system if it still exists. We're okay to - // ignore a failure here. - computeSystem.Terminate() - return nil, makeSystemError(computeSystem, operation, "", err, nil) - } - } - - events, err := processAsyncHcsResult(createError, resultp, computeSystem.callbackNumber, hcsNotificationSystemCreateCompleted, &timeout.SystemCreate) - if err != nil { - if err == ErrTimeout { - // Terminate the compute system if it still exists. We're okay to - // ignore a failure here. - computeSystem.Terminate() - } - return nil, makeSystemError(computeSystem, operation, hcsDocument, err, events) - } - - return computeSystem, nil -} - -// OpenComputeSystem opens an existing compute system by ID. -func OpenComputeSystem(id string) (_ *System, err error) { - operation := "hcsshim::OpenComputeSystem" - - computeSystem := newSystem(id) - computeSystem.logOperationBegin(operation) - defer func() { - if IsNotExist(err) { - computeSystem.logOperationEnd(operation, nil) - } else { - computeSystem.logOperationEnd(operation, err) - } - }() - - var ( - handle hcsSystem - resultp *uint16 - ) - err = hcsOpenComputeSystem(id, &handle, &resultp) - events := processHcsResult(resultp) - if err != nil { - return nil, makeSystemError(computeSystem, operation, "", err, events) - } - - computeSystem.handle = handle - - if err = computeSystem.registerCallback(); err != nil { - return nil, makeSystemError(computeSystem, operation, "", err, nil) - } - - return computeSystem, nil -} - -// GetComputeSystems gets a list of the compute systems on the system that match the query -func GetComputeSystems(q schema1.ComputeSystemQuery) (_ []schema1.ContainerProperties, err error) { - operation := "hcsshim::GetComputeSystems" - fields := logrus.Fields{} - logOperationBegin( - fields, - operation+" - Begin Operation") - - defer func() { - var result string - if err == nil { - result = "Success" - } else { - result = "Error" - } - - logOperationEnd( - fields, - operation+" - End Operation - "+result, - err) - }() - - queryb, err := json.Marshal(q) - if err != nil { - return nil, err - } - - query := string(queryb) - - logrus.WithFields(fields). - WithField(logfields.JSON, query). - Debug("HCS ComputeSystem Query") - - var ( - resultp *uint16 - computeSystemsp *uint16 - ) - - syscallWatcher(fields, func() { - err = hcsEnumerateComputeSystems(query, &computeSystemsp, &resultp) - }) - events := processHcsResult(resultp) - if err != nil { - return nil, &HcsError{Op: operation, Err: err, Events: events} - } - - if computeSystemsp == nil { - return nil, ErrUnexpectedValue - } - computeSystemsRaw := interop.ConvertAndFreeCoTaskMemBytes(computeSystemsp) - computeSystems := []schema1.ContainerProperties{} - if err = json.Unmarshal(computeSystemsRaw, &computeSystems); err != nil { - return nil, err - } - - return computeSystems, nil -} - -// Start synchronously starts the computeSystem. -func (computeSystem *System) Start() (err error) { - computeSystem.handleLock.RLock() - defer computeSystem.handleLock.RUnlock() - - operation := "hcsshim::ComputeSystem::Start" - computeSystem.logOperationBegin(operation) - defer func() { computeSystem.logOperationEnd(operation, err) }() - - if computeSystem.handle == 0 { - return makeSystemError(computeSystem, "Start", "", ErrAlreadyClosed, nil) - } - - // This is a very simple backoff-retry loop to limit the number - // of parallel container starts if environment variable - // HCSSHIM_MAX_PARALLEL_START is set to a positive integer. - // It should generally only be used as a workaround to various - // platform issues that exist between RS1 and RS4 as of Aug 2018 - if currentContainerStarts.maxParallel > 0 { - for { - currentContainerStarts.Lock() - if currentContainerStarts.inProgress < currentContainerStarts.maxParallel { - currentContainerStarts.inProgress++ - currentContainerStarts.Unlock() - break - } - if currentContainerStarts.inProgress == currentContainerStarts.maxParallel { - currentContainerStarts.Unlock() - time.Sleep(100 * time.Millisecond) - } - } - // Make sure we decrement the count when we are done. - defer func() { - currentContainerStarts.Lock() - currentContainerStarts.inProgress-- - currentContainerStarts.Unlock() - }() - } - - var resultp *uint16 - syscallWatcher(computeSystem.logctx, func() { - err = hcsStartComputeSystem(computeSystem.handle, "", &resultp) - }) - events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemStartCompleted, &timeout.SystemStart) - if err != nil { - return makeSystemError(computeSystem, "Start", "", err, events) - } - - return nil -} - -// ID returns the compute system's identifier. -func (computeSystem *System) ID() string { - return computeSystem.id -} - -// Shutdown requests a compute system shutdown, if IsPending() on the error returned is true, -// it may not actually be shut down until Wait() succeeds. -func (computeSystem *System) Shutdown() (err error) { - computeSystem.handleLock.RLock() - defer computeSystem.handleLock.RUnlock() - - operation := "hcsshim::ComputeSystem::Shutdown" - computeSystem.logOperationBegin(operation) - defer func() { - if IsAlreadyStopped(err) { - computeSystem.logOperationEnd(operation, nil) - } else { - computeSystem.logOperationEnd(operation, err) - } - }() - - if computeSystem.handle == 0 { - return makeSystemError(computeSystem, "Shutdown", "", ErrAlreadyClosed, nil) - } - - var resultp *uint16 - syscallWatcher(computeSystem.logctx, func() { - err = hcsShutdownComputeSystem(computeSystem.handle, "", &resultp) - }) - events := processHcsResult(resultp) - if err != nil { - return makeSystemError(computeSystem, "Shutdown", "", err, events) - } - - return nil -} - -// Terminate requests a compute system terminate, if IsPending() on the error returned is true, -// it may not actually be shut down until Wait() succeeds. -func (computeSystem *System) Terminate() (err error) { - computeSystem.handleLock.RLock() - defer computeSystem.handleLock.RUnlock() - - operation := "hcsshim::ComputeSystem::Terminate" - computeSystem.logOperationBegin(operation) - defer func() { - if IsPending(err) { - computeSystem.logOperationEnd(operation, nil) - } else { - computeSystem.logOperationEnd(operation, err) - } - }() - - if computeSystem.handle == 0 { - return makeSystemError(computeSystem, "Terminate", "", ErrAlreadyClosed, nil) - } - - var resultp *uint16 - syscallWatcher(computeSystem.logctx, func() { - err = hcsTerminateComputeSystem(computeSystem.handle, "", &resultp) - }) - events := processHcsResult(resultp) - if err != nil && err != ErrVmcomputeAlreadyStopped { - return makeSystemError(computeSystem, "Terminate", "", err, events) - } - - return nil -} - -// Wait synchronously waits for the compute system to shutdown or terminate. -func (computeSystem *System) Wait() (err error) { - operation := "hcsshim::ComputeSystem::Wait" - computeSystem.logOperationBegin(operation) - defer func() { computeSystem.logOperationEnd(operation, err) }() - - err = waitForNotification(computeSystem.callbackNumber, hcsNotificationSystemExited, nil) - if err != nil { - return makeSystemError(computeSystem, "Wait", "", err, nil) - } - - return nil -} - -// WaitExpectedError synchronously waits for the compute system to shutdown or -// terminate, and ignores the passed error if it occurs. -func (computeSystem *System) WaitExpectedError(expected error) (err error) { - operation := "hcsshim::ComputeSystem::WaitExpectedError" - computeSystem.logOperationBegin(operation) - defer func() { computeSystem.logOperationEnd(operation, err) }() - - err = waitForNotification(computeSystem.callbackNumber, hcsNotificationSystemExited, nil) - if err != nil && getInnerError(err) != expected { - return makeSystemError(computeSystem, "WaitExpectedError", "", err, nil) - } - - return nil -} - -// WaitTimeout synchronously waits for the compute system to terminate or the duration to elapse. -// If the timeout expires, IsTimeout(err) == true -func (computeSystem *System) WaitTimeout(timeout time.Duration) (err error) { - operation := "hcsshim::ComputeSystem::WaitTimeout" - computeSystem.logOperationBegin(operation) - defer func() { computeSystem.logOperationEnd(operation, err) }() - - err = waitForNotification(computeSystem.callbackNumber, hcsNotificationSystemExited, &timeout) - if err != nil { - return makeSystemError(computeSystem, "WaitTimeout", "", err, nil) - } - - return nil -} - -func (computeSystem *System) Properties(types ...schema1.PropertyType) (_ *schema1.ContainerProperties, err error) { - computeSystem.handleLock.RLock() - defer computeSystem.handleLock.RUnlock() - - operation := "hcsshim::ComputeSystem::Properties" - computeSystem.logOperationBegin(operation) - defer func() { computeSystem.logOperationEnd(operation, err) }() - - queryj, err := json.Marshal(schema1.PropertyQuery{types}) - if err != nil { - return nil, makeSystemError(computeSystem, "Properties", "", err, nil) - } - - logrus.WithFields(computeSystem.logctx). - WithField(logfields.JSON, queryj). - Debug("HCS ComputeSystem Properties Query") - - var resultp, propertiesp *uint16 - syscallWatcher(computeSystem.logctx, func() { - err = hcsGetComputeSystemProperties(computeSystem.handle, string(queryj), &propertiesp, &resultp) - }) - events := processHcsResult(resultp) - if err != nil { - return nil, makeSystemError(computeSystem, "Properties", "", err, events) - } - - if propertiesp == nil { - return nil, ErrUnexpectedValue - } - propertiesRaw := interop.ConvertAndFreeCoTaskMemBytes(propertiesp) - properties := &schema1.ContainerProperties{} - if err := json.Unmarshal(propertiesRaw, properties); err != nil { - return nil, makeSystemError(computeSystem, "Properties", "", err, nil) - } - - return properties, nil -} - -// Pause pauses the execution of the computeSystem. This feature is not enabled in TP5. -func (computeSystem *System) Pause() (err error) { - computeSystem.handleLock.RLock() - defer computeSystem.handleLock.RUnlock() - - operation := "hcsshim::ComputeSystem::Pause" - computeSystem.logOperationBegin(operation) - defer func() { computeSystem.logOperationEnd(operation, err) }() - - if computeSystem.handle == 0 { - return makeSystemError(computeSystem, "Pause", "", ErrAlreadyClosed, nil) - } - - var resultp *uint16 - syscallWatcher(computeSystem.logctx, func() { - err = hcsPauseComputeSystem(computeSystem.handle, "", &resultp) - }) - events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemPauseCompleted, &timeout.SystemPause) - if err != nil { - return makeSystemError(computeSystem, "Pause", "", err, events) - } - - return nil -} - -// Resume resumes the execution of the computeSystem. This feature is not enabled in TP5. -func (computeSystem *System) Resume() (err error) { - computeSystem.handleLock.RLock() - defer computeSystem.handleLock.RUnlock() - - operation := "hcsshim::ComputeSystem::Resume" - computeSystem.logOperationBegin(operation) - defer func() { computeSystem.logOperationEnd(operation, err) }() - - if computeSystem.handle == 0 { - return makeSystemError(computeSystem, "Resume", "", ErrAlreadyClosed, nil) - } - - var resultp *uint16 - syscallWatcher(computeSystem.logctx, func() { - err = hcsResumeComputeSystem(computeSystem.handle, "", &resultp) - }) - events, err := processAsyncHcsResult(err, resultp, computeSystem.callbackNumber, hcsNotificationSystemResumeCompleted, &timeout.SystemResume) - if err != nil { - return makeSystemError(computeSystem, "Resume", "", err, events) - } - - return nil -} - -// CreateProcess launches a new process within the computeSystem. -func (computeSystem *System) CreateProcess(c interface{}) (_ *Process, err error) { - computeSystem.handleLock.RLock() - defer computeSystem.handleLock.RUnlock() - - operation := "hcsshim::ComputeSystem::CreateProcess" - computeSystem.logOperationBegin(operation) - defer func() { computeSystem.logOperationEnd(operation, err) }() - - var ( - processInfo hcsProcessInformation - processHandle hcsProcess - resultp *uint16 - ) - - if computeSystem.handle == 0 { - return nil, makeSystemError(computeSystem, "CreateProcess", "", ErrAlreadyClosed, nil) - } - - configurationb, err := json.Marshal(c) - if err != nil { - return nil, makeSystemError(computeSystem, "CreateProcess", "", err, nil) - } - - configuration := string(configurationb) - - logrus.WithFields(computeSystem.logctx). - WithField(logfields.JSON, configuration). - Debug("HCS ComputeSystem Process Document") - - syscallWatcher(computeSystem.logctx, func() { - err = hcsCreateProcess(computeSystem.handle, configuration, &processInfo, &processHandle, &resultp) - }) - events := processHcsResult(resultp) - if err != nil { - return nil, makeSystemError(computeSystem, "CreateProcess", configuration, err, events) - } - - logrus.WithFields(computeSystem.logctx). - WithField(logfields.ProcessID, processInfo.ProcessId). - Debug("HCS ComputeSystem CreateProcess PID") - - process := newProcess(processHandle, int(processInfo.ProcessId), computeSystem) - process.cachedPipes = &cachedPipes{ - stdIn: processInfo.StdInput, - stdOut: processInfo.StdOutput, - stdErr: processInfo.StdError, - } - - if err = process.registerCallback(); err != nil { - return nil, makeSystemError(computeSystem, "CreateProcess", "", err, nil) - } - - return process, nil -} - -// OpenProcess gets an interface to an existing process within the computeSystem. -func (computeSystem *System) OpenProcess(pid int) (_ *Process, err error) { - computeSystem.handleLock.RLock() - defer computeSystem.handleLock.RUnlock() - - // Add PID for the context of this operation - computeSystem.logctx[logfields.ProcessID] = pid - defer delete(computeSystem.logctx, logfields.ProcessID) - - operation := "hcsshim::ComputeSystem::OpenProcess" - computeSystem.logOperationBegin(operation) - defer func() { computeSystem.logOperationEnd(operation, err) }() - - var ( - processHandle hcsProcess - resultp *uint16 - ) - - if computeSystem.handle == 0 { - return nil, makeSystemError(computeSystem, "OpenProcess", "", ErrAlreadyClosed, nil) - } - - syscallWatcher(computeSystem.logctx, func() { - err = hcsOpenProcess(computeSystem.handle, uint32(pid), &processHandle, &resultp) - }) - events := processHcsResult(resultp) - if err != nil { - return nil, makeSystemError(computeSystem, "OpenProcess", "", err, events) - } - - process := newProcess(processHandle, pid, computeSystem) - if err = process.registerCallback(); err != nil { - return nil, makeSystemError(computeSystem, "OpenProcess", "", err, nil) - } - - return process, nil -} - -// Close cleans up any state associated with the compute system but does not terminate or wait for it. -func (computeSystem *System) Close() (err error) { - computeSystem.handleLock.Lock() - defer computeSystem.handleLock.Unlock() - - operation := "hcsshim::ComputeSystem::Close" - computeSystem.logOperationBegin(operation) - defer func() { computeSystem.logOperationEnd(operation, err) }() - - // Don't double free this - if computeSystem.handle == 0 { - return nil - } - - if err = computeSystem.unregisterCallback(); err != nil { - return makeSystemError(computeSystem, "Close", "", err, nil) - } - - syscallWatcher(computeSystem.logctx, func() { - err = hcsCloseComputeSystem(computeSystem.handle) - }) - if err != nil { - return makeSystemError(computeSystem, "Close", "", err, nil) - } - - computeSystem.handle = 0 - - return nil -} - -func (computeSystem *System) registerCallback() error { - context := ¬ifcationWatcherContext{ - channels: newChannels(), - } - - callbackMapLock.Lock() - callbackNumber := nextCallback - nextCallback++ - callbackMap[callbackNumber] = context - callbackMapLock.Unlock() - - var callbackHandle hcsCallback - err := hcsRegisterComputeSystemCallback(computeSystem.handle, notificationWatcherCallback, callbackNumber, &callbackHandle) - if err != nil { - return err - } - context.handle = callbackHandle - computeSystem.callbackNumber = callbackNumber - - return nil -} - -func (computeSystem *System) unregisterCallback() error { - callbackNumber := computeSystem.callbackNumber - - callbackMapLock.RLock() - context := callbackMap[callbackNumber] - callbackMapLock.RUnlock() - - if context == nil { - return nil - } - - handle := context.handle - - if handle == 0 { - return nil - } - - // hcsUnregisterComputeSystemCallback has its own syncronization - // to wait for all callbacks to complete. We must NOT hold the callbackMapLock. - err := hcsUnregisterComputeSystemCallback(handle) - if err != nil { - return err - } - - closeChannels(context.channels) - - callbackMapLock.Lock() - callbackMap[callbackNumber] = nil - callbackMapLock.Unlock() - - handle = 0 - - return nil -} - -// Modify the System by sending a request to HCS -func (computeSystem *System) Modify(config interface{}) (err error) { - computeSystem.handleLock.RLock() - defer computeSystem.handleLock.RUnlock() - - operation := "hcsshim::ComputeSystem::Modify" - computeSystem.logOperationBegin(operation) - defer func() { computeSystem.logOperationEnd(operation, err) }() - - if computeSystem.handle == 0 { - return makeSystemError(computeSystem, "Modify", "", ErrAlreadyClosed, nil) - } - - requestJSON, err := json.Marshal(config) - if err != nil { - return err - } - - requestString := string(requestJSON) - - logrus.WithFields(computeSystem.logctx). - WithField(logfields.JSON, requestString). - Debug("HCS ComputeSystem Modify Document") - - var resultp *uint16 - syscallWatcher(computeSystem.logctx, func() { - err = hcsModifyComputeSystem(computeSystem.handle, requestString, &resultp) - }) - events := processHcsResult(resultp) - if err != nil { - return makeSystemError(computeSystem, "Modify", requestString, err, events) - } - - return nil -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hcs/utils.go b/vendor/github.com/Microsoft/hcsshim/internal/hcs/utils.go deleted file mode 100644 index a638677ed..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/hcs/utils.go +++ /dev/null @@ -1,33 +0,0 @@ -package hcs - -import ( - "io" - "syscall" - - "github.com/Microsoft/go-winio" -) - -// makeOpenFiles calls winio.MakeOpenFile for each handle in a slice but closes all the handles -// if there is an error. -func makeOpenFiles(hs []syscall.Handle) (_ []io.ReadWriteCloser, err error) { - fs := make([]io.ReadWriteCloser, len(hs)) - for i, h := range hs { - if h != syscall.Handle(0) { - if err == nil { - fs[i], err = winio.MakeOpenFile(h) - } - if err != nil { - syscall.Close(h) - } - } - } - if err != nil { - for _, f := range fs { - if f != nil { - f.Close() - } - } - return nil, err - } - return fs, nil -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hcs/waithelper.go b/vendor/github.com/Microsoft/hcsshim/internal/hcs/waithelper.go deleted file mode 100644 index 91e212c57..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/hcs/waithelper.go +++ /dev/null @@ -1,63 +0,0 @@ -package hcs - -import ( - "time" - - "github.com/sirupsen/logrus" -) - -func processAsyncHcsResult(err error, resultp *uint16, callbackNumber uintptr, expectedNotification hcsNotification, timeout *time.Duration) ([]ErrorEvent, error) { - events := processHcsResult(resultp) - if IsPending(err) { - return nil, waitForNotification(callbackNumber, expectedNotification, timeout) - } - - return events, err -} - -func waitForNotification(callbackNumber uintptr, expectedNotification hcsNotification, timeout *time.Duration) error { - callbackMapLock.RLock() - channels := callbackMap[callbackNumber].channels - callbackMapLock.RUnlock() - - expectedChannel := channels[expectedNotification] - if expectedChannel == nil { - logrus.Errorf("unknown notification type in waitForNotification %x", expectedNotification) - return ErrInvalidNotificationType - } - - var c <-chan time.Time - if timeout != nil { - timer := time.NewTimer(*timeout) - c = timer.C - defer timer.Stop() - } - - select { - case err, ok := <-expectedChannel: - if !ok { - return ErrHandleClose - } - return err - case err, ok := <-channels[hcsNotificationSystemExited]: - if !ok { - return ErrHandleClose - } - // If the expected notification is hcsNotificationSystemExited which of the two selects - // chosen is random. Return the raw error if hcsNotificationSystemExited is expected - if channels[hcsNotificationSystemExited] == expectedChannel { - return err - } - return ErrUnexpectedContainerExit - case _, ok := <-channels[hcsNotificationServiceDisconnect]: - if !ok { - return ErrHandleClose - } - // hcsNotificationServiceDisconnect should never be an expected notification - // it does not need the same handling as hcsNotificationSystemExited - return ErrUnexpectedProcessAbort - case <-c: - return ErrTimeout - } - return nil -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hcs/watcher.go b/vendor/github.com/Microsoft/hcsshim/internal/hcs/watcher.go deleted file mode 100644 index f85ed3187..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/hcs/watcher.go +++ /dev/null @@ -1,41 +0,0 @@ -package hcs - -import ( - "context" - - "github.com/Microsoft/hcsshim/internal/logfields" - "github.com/Microsoft/hcsshim/internal/timeout" - "github.com/sirupsen/logrus" -) - -// syscallWatcher is used as a very simple goroutine around calls into -// the platform. In some cases, we have seen HCS APIs not returning due to -// various bugs, and the goroutine making the syscall ends up not returning, -// prior to its async callback. By spinning up a syscallWatcher, it allows -// us to at least log a warning if a syscall doesn't complete in a reasonable -// amount of time. -// -// Usage is: -// -// syscallWatcher(logContext, func() { -// err = (args...) -// }) -// - -func syscallWatcher(logContext logrus.Fields, syscallLambda func()) { - ctx, cancel := context.WithTimeout(context.Background(), timeout.SyscallWatcher) - defer cancel() - go watchFunc(ctx, logContext) - syscallLambda() -} - -func watchFunc(ctx context.Context, logContext logrus.Fields) { - select { - case <-ctx.Done(): - if ctx.Err() != context.Canceled { - logrus.WithFields(logContext). - WithField(logfields.Timeout, timeout.SyscallWatcher). - Warning("Syscall did not complete within operation timeout. This may indicate a platform issue. If it appears to be making no forward progress, obtain the stacks and see if there is a syscall stuck in the platform API for a significant length of time.") - } - } -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hcs/zsyscall_windows.go b/vendor/github.com/Microsoft/hcsshim/internal/hcs/zsyscall_windows.go deleted file mode 100644 index fcd5cdc87..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/hcs/zsyscall_windows.go +++ /dev/null @@ -1,533 +0,0 @@ -// Code generated mksyscall_windows.exe DO NOT EDIT - -package hcs - -import ( - "syscall" - "unsafe" - - "golang.org/x/sys/windows" -) - -var _ unsafe.Pointer - -// Do the interface allocations only once for common -// Errno values. -const ( - errnoERROR_IO_PENDING = 997 -) - -var ( - errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING) -) - -// errnoErr returns common boxed Errno values, to prevent -// allocations at runtime. -func errnoErr(e syscall.Errno) error { - switch e { - case 0: - return nil - case errnoERROR_IO_PENDING: - return errERROR_IO_PENDING - } - // TODO: add more here, after collecting data on the common - // error values see on Windows. (perhaps when running - // all.bat?) - return e -} - -var ( - modvmcompute = windows.NewLazySystemDLL("vmcompute.dll") - - procHcsEnumerateComputeSystems = modvmcompute.NewProc("HcsEnumerateComputeSystems") - procHcsCreateComputeSystem = modvmcompute.NewProc("HcsCreateComputeSystem") - procHcsOpenComputeSystem = modvmcompute.NewProc("HcsOpenComputeSystem") - procHcsCloseComputeSystem = modvmcompute.NewProc("HcsCloseComputeSystem") - procHcsStartComputeSystem = modvmcompute.NewProc("HcsStartComputeSystem") - procHcsShutdownComputeSystem = modvmcompute.NewProc("HcsShutdownComputeSystem") - procHcsTerminateComputeSystem = modvmcompute.NewProc("HcsTerminateComputeSystem") - procHcsPauseComputeSystem = modvmcompute.NewProc("HcsPauseComputeSystem") - procHcsResumeComputeSystem = modvmcompute.NewProc("HcsResumeComputeSystem") - procHcsGetComputeSystemProperties = modvmcompute.NewProc("HcsGetComputeSystemProperties") - procHcsModifyComputeSystem = modvmcompute.NewProc("HcsModifyComputeSystem") - procHcsRegisterComputeSystemCallback = modvmcompute.NewProc("HcsRegisterComputeSystemCallback") - procHcsUnregisterComputeSystemCallback = modvmcompute.NewProc("HcsUnregisterComputeSystemCallback") - procHcsCreateProcess = modvmcompute.NewProc("HcsCreateProcess") - procHcsOpenProcess = modvmcompute.NewProc("HcsOpenProcess") - procHcsCloseProcess = modvmcompute.NewProc("HcsCloseProcess") - procHcsTerminateProcess = modvmcompute.NewProc("HcsTerminateProcess") - - procHcsGetProcessInfo = modvmcompute.NewProc("HcsGetProcessInfo") - procHcsGetProcessProperties = modvmcompute.NewProc("HcsGetProcessProperties") - procHcsModifyProcess = modvmcompute.NewProc("HcsModifyProcess") - procHcsGetServiceProperties = modvmcompute.NewProc("HcsGetServiceProperties") - procHcsRegisterProcessCallback = modvmcompute.NewProc("HcsRegisterProcessCallback") - procHcsUnregisterProcessCallback = modvmcompute.NewProc("HcsUnregisterProcessCallback") -) - -func hcsEnumerateComputeSystems(query string, computeSystems **uint16, result **uint16) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(query) - if hr != nil { - return - } - return _hcsEnumerateComputeSystems(_p0, computeSystems, result) -} - -func _hcsEnumerateComputeSystems(query *uint16, computeSystems **uint16, result **uint16) (hr error) { - if hr = procHcsEnumerateComputeSystems.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procHcsEnumerateComputeSystems.Addr(), 3, uintptr(unsafe.Pointer(query)), uintptr(unsafe.Pointer(computeSystems)), uintptr(unsafe.Pointer(result))) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func hcsCreateComputeSystem(id string, configuration string, identity syscall.Handle, computeSystem *hcsSystem, result **uint16) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(id) - if hr != nil { - return - } - var _p1 *uint16 - _p1, hr = syscall.UTF16PtrFromString(configuration) - if hr != nil { - return - } - return _hcsCreateComputeSystem(_p0, _p1, identity, computeSystem, result) -} - -func _hcsCreateComputeSystem(id *uint16, configuration *uint16, identity syscall.Handle, computeSystem *hcsSystem, result **uint16) (hr error) { - if hr = procHcsCreateComputeSystem.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall6(procHcsCreateComputeSystem.Addr(), 5, uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(configuration)), uintptr(identity), uintptr(unsafe.Pointer(computeSystem)), uintptr(unsafe.Pointer(result)), 0) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func hcsOpenComputeSystem(id string, computeSystem *hcsSystem, result **uint16) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(id) - if hr != nil { - return - } - return _hcsOpenComputeSystem(_p0, computeSystem, result) -} - -func _hcsOpenComputeSystem(id *uint16, computeSystem *hcsSystem, result **uint16) (hr error) { - if hr = procHcsOpenComputeSystem.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procHcsOpenComputeSystem.Addr(), 3, uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(computeSystem)), uintptr(unsafe.Pointer(result))) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func hcsCloseComputeSystem(computeSystem hcsSystem) (hr error) { - if hr = procHcsCloseComputeSystem.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procHcsCloseComputeSystem.Addr(), 1, uintptr(computeSystem), 0, 0) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func hcsStartComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(options) - if hr != nil { - return - } - return _hcsStartComputeSystem(computeSystem, _p0, result) -} - -func _hcsStartComputeSystem(computeSystem hcsSystem, options *uint16, result **uint16) (hr error) { - if hr = procHcsStartComputeSystem.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procHcsStartComputeSystem.Addr(), 3, uintptr(computeSystem), uintptr(unsafe.Pointer(options)), uintptr(unsafe.Pointer(result))) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func hcsShutdownComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(options) - if hr != nil { - return - } - return _hcsShutdownComputeSystem(computeSystem, _p0, result) -} - -func _hcsShutdownComputeSystem(computeSystem hcsSystem, options *uint16, result **uint16) (hr error) { - if hr = procHcsShutdownComputeSystem.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procHcsShutdownComputeSystem.Addr(), 3, uintptr(computeSystem), uintptr(unsafe.Pointer(options)), uintptr(unsafe.Pointer(result))) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func hcsTerminateComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(options) - if hr != nil { - return - } - return _hcsTerminateComputeSystem(computeSystem, _p0, result) -} - -func _hcsTerminateComputeSystem(computeSystem hcsSystem, options *uint16, result **uint16) (hr error) { - if hr = procHcsTerminateComputeSystem.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procHcsTerminateComputeSystem.Addr(), 3, uintptr(computeSystem), uintptr(unsafe.Pointer(options)), uintptr(unsafe.Pointer(result))) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func hcsPauseComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(options) - if hr != nil { - return - } - return _hcsPauseComputeSystem(computeSystem, _p0, result) -} - -func _hcsPauseComputeSystem(computeSystem hcsSystem, options *uint16, result **uint16) (hr error) { - if hr = procHcsPauseComputeSystem.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procHcsPauseComputeSystem.Addr(), 3, uintptr(computeSystem), uintptr(unsafe.Pointer(options)), uintptr(unsafe.Pointer(result))) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func hcsResumeComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(options) - if hr != nil { - return - } - return _hcsResumeComputeSystem(computeSystem, _p0, result) -} - -func _hcsResumeComputeSystem(computeSystem hcsSystem, options *uint16, result **uint16) (hr error) { - if hr = procHcsResumeComputeSystem.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procHcsResumeComputeSystem.Addr(), 3, uintptr(computeSystem), uintptr(unsafe.Pointer(options)), uintptr(unsafe.Pointer(result))) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func hcsGetComputeSystemProperties(computeSystem hcsSystem, propertyQuery string, properties **uint16, result **uint16) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(propertyQuery) - if hr != nil { - return - } - return _hcsGetComputeSystemProperties(computeSystem, _p0, properties, result) -} - -func _hcsGetComputeSystemProperties(computeSystem hcsSystem, propertyQuery *uint16, properties **uint16, result **uint16) (hr error) { - if hr = procHcsGetComputeSystemProperties.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall6(procHcsGetComputeSystemProperties.Addr(), 4, uintptr(computeSystem), uintptr(unsafe.Pointer(propertyQuery)), uintptr(unsafe.Pointer(properties)), uintptr(unsafe.Pointer(result)), 0, 0) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func hcsModifyComputeSystem(computeSystem hcsSystem, configuration string, result **uint16) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(configuration) - if hr != nil { - return - } - return _hcsModifyComputeSystem(computeSystem, _p0, result) -} - -func _hcsModifyComputeSystem(computeSystem hcsSystem, configuration *uint16, result **uint16) (hr error) { - if hr = procHcsModifyComputeSystem.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procHcsModifyComputeSystem.Addr(), 3, uintptr(computeSystem), uintptr(unsafe.Pointer(configuration)), uintptr(unsafe.Pointer(result))) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func hcsRegisterComputeSystemCallback(computeSystem hcsSystem, callback uintptr, context uintptr, callbackHandle *hcsCallback) (hr error) { - if hr = procHcsRegisterComputeSystemCallback.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall6(procHcsRegisterComputeSystemCallback.Addr(), 4, uintptr(computeSystem), uintptr(callback), uintptr(context), uintptr(unsafe.Pointer(callbackHandle)), 0, 0) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func hcsUnregisterComputeSystemCallback(callbackHandle hcsCallback) (hr error) { - if hr = procHcsUnregisterComputeSystemCallback.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procHcsUnregisterComputeSystemCallback.Addr(), 1, uintptr(callbackHandle), 0, 0) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func hcsCreateProcess(computeSystem hcsSystem, processParameters string, processInformation *hcsProcessInformation, process *hcsProcess, result **uint16) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(processParameters) - if hr != nil { - return - } - return _hcsCreateProcess(computeSystem, _p0, processInformation, process, result) -} - -func _hcsCreateProcess(computeSystem hcsSystem, processParameters *uint16, processInformation *hcsProcessInformation, process *hcsProcess, result **uint16) (hr error) { - if hr = procHcsCreateProcess.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall6(procHcsCreateProcess.Addr(), 5, uintptr(computeSystem), uintptr(unsafe.Pointer(processParameters)), uintptr(unsafe.Pointer(processInformation)), uintptr(unsafe.Pointer(process)), uintptr(unsafe.Pointer(result)), 0) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func hcsOpenProcess(computeSystem hcsSystem, pid uint32, process *hcsProcess, result **uint16) (hr error) { - if hr = procHcsOpenProcess.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall6(procHcsOpenProcess.Addr(), 4, uintptr(computeSystem), uintptr(pid), uintptr(unsafe.Pointer(process)), uintptr(unsafe.Pointer(result)), 0, 0) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func hcsCloseProcess(process hcsProcess) (hr error) { - if hr = procHcsCloseProcess.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procHcsCloseProcess.Addr(), 1, uintptr(process), 0, 0) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func hcsTerminateProcess(process hcsProcess, result **uint16) (hr error) { - if hr = procHcsTerminateProcess.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procHcsTerminateProcess.Addr(), 2, uintptr(process), uintptr(unsafe.Pointer(result)), 0) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func hcsSignalProcess(process hcsProcess, options string, result **uint16) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(options) - if hr != nil { - return - } - return _hcsSignalProcess(process, _p0, result) -} - -func _hcsSignalProcess(process hcsProcess, options *uint16, result **uint16) (hr error) { - if hr = procHcsTerminateProcess.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procHcsTerminateProcess.Addr(), 3, uintptr(process), uintptr(unsafe.Pointer(options)), uintptr(unsafe.Pointer(result))) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func hcsGetProcessInfo(process hcsProcess, processInformation *hcsProcessInformation, result **uint16) (hr error) { - if hr = procHcsGetProcessInfo.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procHcsGetProcessInfo.Addr(), 3, uintptr(process), uintptr(unsafe.Pointer(processInformation)), uintptr(unsafe.Pointer(result))) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func hcsGetProcessProperties(process hcsProcess, processProperties **uint16, result **uint16) (hr error) { - if hr = procHcsGetProcessProperties.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procHcsGetProcessProperties.Addr(), 3, uintptr(process), uintptr(unsafe.Pointer(processProperties)), uintptr(unsafe.Pointer(result))) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func hcsModifyProcess(process hcsProcess, settings string, result **uint16) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(settings) - if hr != nil { - return - } - return _hcsModifyProcess(process, _p0, result) -} - -func _hcsModifyProcess(process hcsProcess, settings *uint16, result **uint16) (hr error) { - if hr = procHcsModifyProcess.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procHcsModifyProcess.Addr(), 3, uintptr(process), uintptr(unsafe.Pointer(settings)), uintptr(unsafe.Pointer(result))) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func hcsGetServiceProperties(propertyQuery string, properties **uint16, result **uint16) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(propertyQuery) - if hr != nil { - return - } - return _hcsGetServiceProperties(_p0, properties, result) -} - -func _hcsGetServiceProperties(propertyQuery *uint16, properties **uint16, result **uint16) (hr error) { - if hr = procHcsGetServiceProperties.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procHcsGetServiceProperties.Addr(), 3, uintptr(unsafe.Pointer(propertyQuery)), uintptr(unsafe.Pointer(properties)), uintptr(unsafe.Pointer(result))) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func hcsRegisterProcessCallback(process hcsProcess, callback uintptr, context uintptr, callbackHandle *hcsCallback) (hr error) { - if hr = procHcsRegisterProcessCallback.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall6(procHcsRegisterProcessCallback.Addr(), 4, uintptr(process), uintptr(callback), uintptr(context), uintptr(unsafe.Pointer(callbackHandle)), 0, 0) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func hcsUnregisterProcessCallback(callbackHandle hcsCallback) (hr error) { - if hr = procHcsUnregisterProcessCallback.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procHcsUnregisterProcessCallback.Addr(), 1, uintptr(callbackHandle), 0, 0) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hcserror/hcserror.go b/vendor/github.com/Microsoft/hcsshim/internal/hcserror/hcserror.go deleted file mode 100644 index 921c2c855..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/hcserror/hcserror.go +++ /dev/null @@ -1,47 +0,0 @@ -package hcserror - -import ( - "fmt" - "syscall" -) - -const ERROR_GEN_FAILURE = syscall.Errno(31) - -type HcsError struct { - title string - rest string - Err error -} - -func (e *HcsError) Error() string { - s := e.title - if len(s) > 0 && s[len(s)-1] != ' ' { - s += " " - } - s += fmt.Sprintf("failed in Win32: %s (0x%x)", e.Err, Win32FromError(e.Err)) - if e.rest != "" { - if e.rest[0] != ' ' { - s += " " - } - s += e.rest - } - return s -} - -func New(err error, title, rest string) error { - // Pass through DLL errors directly since they do not originate from HCS. - if _, ok := err.(*syscall.DLLError); ok { - return err - } - return &HcsError{title, rest, err} -} - -func Win32FromError(err error) uint32 { - if herr, ok := err.(*HcsError); ok { - return Win32FromError(herr.Err) - } - if code, ok := err.(syscall.Errno); ok { - return uint32(code) - } - return uint32(ERROR_GEN_FAILURE) -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hns/hns.go b/vendor/github.com/Microsoft/hcsshim/internal/hns/hns.go deleted file mode 100644 index b2e475f53..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/hns/hns.go +++ /dev/null @@ -1,23 +0,0 @@ -package hns - -import "fmt" - -//go:generate go run ../../mksyscall_windows.go -output zsyscall_windows.go hns.go - -//sys _hnsCall(method string, path string, object string, response **uint16) (hr error) = vmcompute.HNSCall? - -type EndpointNotFoundError struct { - EndpointName string -} - -func (e EndpointNotFoundError) Error() string { - return fmt.Sprintf("Endpoint %s not found", e.EndpointName) -} - -type NetworkNotFoundError struct { - NetworkName string -} - -func (e NetworkNotFoundError) Error() string { - return fmt.Sprintf("Network %s not found", e.NetworkName) -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hns/hnsendpoint.go b/vendor/github.com/Microsoft/hcsshim/internal/hns/hnsendpoint.go deleted file mode 100644 index 59ec7004c..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/hns/hnsendpoint.go +++ /dev/null @@ -1,262 +0,0 @@ -package hns - -import ( - "encoding/json" - "net" - - "github.com/sirupsen/logrus" -) - -// HNSEndpoint represents a network endpoint in HNS -type HNSEndpoint struct { - Id string `json:"ID,omitempty"` - Name string `json:",omitempty"` - VirtualNetwork string `json:",omitempty"` - VirtualNetworkName string `json:",omitempty"` - Policies []json.RawMessage `json:",omitempty"` - MacAddress string `json:",omitempty"` - IPAddress net.IP `json:",omitempty"` - DNSSuffix string `json:",omitempty"` - DNSServerList string `json:",omitempty"` - GatewayAddress string `json:",omitempty"` - EnableInternalDNS bool `json:",omitempty"` - DisableICC bool `json:",omitempty"` - PrefixLength uint8 `json:",omitempty"` - IsRemoteEndpoint bool `json:",omitempty"` - EnableLowMetric bool `json:",omitempty"` - Namespace *Namespace `json:",omitempty"` - EncapOverhead uint16 `json:",omitempty"` -} - -//SystemType represents the type of the system on which actions are done -type SystemType string - -// SystemType const -const ( - ContainerType SystemType = "Container" - VirtualMachineType SystemType = "VirtualMachine" - HostType SystemType = "Host" -) - -// EndpointAttachDetachRequest is the structure used to send request to the container to modify the system -// Supported resource types are Network and Request Types are Add/Remove -type EndpointAttachDetachRequest struct { - ContainerID string `json:"ContainerId,omitempty"` - SystemType SystemType `json:"SystemType"` - CompartmentID uint16 `json:"CompartmentId,omitempty"` - VirtualNICName string `json:"VirtualNicName,omitempty"` -} - -// EndpointResquestResponse is object to get the endpoint request response -type EndpointResquestResponse struct { - Success bool - Error string -} - -// HNSEndpointRequest makes a HNS call to modify/query a network endpoint -func HNSEndpointRequest(method, path, request string) (*HNSEndpoint, error) { - endpoint := &HNSEndpoint{} - err := hnsCall(method, "/endpoints/"+path, request, &endpoint) - if err != nil { - return nil, err - } - - return endpoint, nil -} - -// HNSListEndpointRequest makes a HNS call to query the list of available endpoints -func HNSListEndpointRequest() ([]HNSEndpoint, error) { - var endpoint []HNSEndpoint - err := hnsCall("GET", "/endpoints/", "", &endpoint) - if err != nil { - return nil, err - } - - return endpoint, nil -} - -// GetHNSEndpointByID get the Endpoint by ID -func GetHNSEndpointByID(endpointID string) (*HNSEndpoint, error) { - return HNSEndpointRequest("GET", endpointID, "") -} - -// GetHNSEndpointByName gets the endpoint filtered by Name -func GetHNSEndpointByName(endpointName string) (*HNSEndpoint, error) { - hnsResponse, err := HNSListEndpointRequest() - if err != nil { - return nil, err - } - for _, hnsEndpoint := range hnsResponse { - if hnsEndpoint.Name == endpointName { - return &hnsEndpoint, nil - } - } - return nil, EndpointNotFoundError{EndpointName: endpointName} -} - -// Create Endpoint by sending EndpointRequest to HNS. TODO: Create a separate HNS interface to place all these methods -func (endpoint *HNSEndpoint) Create() (*HNSEndpoint, error) { - operation := "Create" - title := "hcsshim::HNSEndpoint::" + operation - logrus.Debugf(title+" id=%s", endpoint.Id) - - jsonString, err := json.Marshal(endpoint) - if err != nil { - return nil, err - } - return HNSEndpointRequest("POST", "", string(jsonString)) -} - -// Delete Endpoint by sending EndpointRequest to HNS -func (endpoint *HNSEndpoint) Delete() (*HNSEndpoint, error) { - operation := "Delete" - title := "hcsshim::HNSEndpoint::" + operation - logrus.Debugf(title+" id=%s", endpoint.Id) - - return HNSEndpointRequest("DELETE", endpoint.Id, "") -} - -// Update Endpoint -func (endpoint *HNSEndpoint) Update() (*HNSEndpoint, error) { - operation := "Update" - title := "hcsshim::HNSEndpoint::" + operation - logrus.Debugf(title+" id=%s", endpoint.Id) - jsonString, err := json.Marshal(endpoint) - if err != nil { - return nil, err - } - err = hnsCall("POST", "/endpoints/"+endpoint.Id, string(jsonString), &endpoint) - - return endpoint, err -} - -// ApplyACLPolicy applies a set of ACL Policies on the Endpoint -func (endpoint *HNSEndpoint) ApplyACLPolicy(policies ...*ACLPolicy) error { - operation := "ApplyACLPolicy" - title := "hcsshim::HNSEndpoint::" + operation - logrus.Debugf(title+" id=%s", endpoint.Id) - - for _, policy := range policies { - if policy == nil { - continue - } - jsonString, err := json.Marshal(policy) - if err != nil { - return err - } - endpoint.Policies = append(endpoint.Policies, jsonString) - } - - _, err := endpoint.Update() - return err -} - -// ContainerAttach attaches an endpoint to container -func (endpoint *HNSEndpoint) ContainerAttach(containerID string, compartmentID uint16) error { - operation := "ContainerAttach" - title := "hcsshim::HNSEndpoint::" + operation - logrus.Debugf(title+" id=%s", endpoint.Id) - - requestMessage := &EndpointAttachDetachRequest{ - ContainerID: containerID, - CompartmentID: compartmentID, - SystemType: ContainerType, - } - response := &EndpointResquestResponse{} - jsonString, err := json.Marshal(requestMessage) - if err != nil { - return err - } - return hnsCall("POST", "/endpoints/"+endpoint.Id+"/attach", string(jsonString), &response) -} - -// ContainerDetach detaches an endpoint from container -func (endpoint *HNSEndpoint) ContainerDetach(containerID string) error { - operation := "ContainerDetach" - title := "hcsshim::HNSEndpoint::" + operation - logrus.Debugf(title+" id=%s", endpoint.Id) - - requestMessage := &EndpointAttachDetachRequest{ - ContainerID: containerID, - SystemType: ContainerType, - } - response := &EndpointResquestResponse{} - - jsonString, err := json.Marshal(requestMessage) - if err != nil { - return err - } - return hnsCall("POST", "/endpoints/"+endpoint.Id+"/detach", string(jsonString), &response) -} - -// HostAttach attaches a nic on the host -func (endpoint *HNSEndpoint) HostAttach(compartmentID uint16) error { - operation := "HostAttach" - title := "hcsshim::HNSEndpoint::" + operation - logrus.Debugf(title+" id=%s", endpoint.Id) - requestMessage := &EndpointAttachDetachRequest{ - CompartmentID: compartmentID, - SystemType: HostType, - } - response := &EndpointResquestResponse{} - - jsonString, err := json.Marshal(requestMessage) - if err != nil { - return err - } - return hnsCall("POST", "/endpoints/"+endpoint.Id+"/attach", string(jsonString), &response) - -} - -// HostDetach detaches a nic on the host -func (endpoint *HNSEndpoint) HostDetach() error { - operation := "HostDetach" - title := "hcsshim::HNSEndpoint::" + operation - logrus.Debugf(title+" id=%s", endpoint.Id) - requestMessage := &EndpointAttachDetachRequest{ - SystemType: HostType, - } - response := &EndpointResquestResponse{} - - jsonString, err := json.Marshal(requestMessage) - if err != nil { - return err - } - return hnsCall("POST", "/endpoints/"+endpoint.Id+"/detach", string(jsonString), &response) -} - -// VirtualMachineNICAttach attaches a endpoint to a virtual machine -func (endpoint *HNSEndpoint) VirtualMachineNICAttach(virtualMachineNICName string) error { - operation := "VirtualMachineNicAttach" - title := "hcsshim::HNSEndpoint::" + operation - logrus.Debugf(title+" id=%s", endpoint.Id) - requestMessage := &EndpointAttachDetachRequest{ - VirtualNICName: virtualMachineNICName, - SystemType: VirtualMachineType, - } - response := &EndpointResquestResponse{} - - jsonString, err := json.Marshal(requestMessage) - if err != nil { - return err - } - return hnsCall("POST", "/endpoints/"+endpoint.Id+"/attach", string(jsonString), &response) -} - -// VirtualMachineNICDetach detaches a endpoint from a virtual machine -func (endpoint *HNSEndpoint) VirtualMachineNICDetach() error { - operation := "VirtualMachineNicDetach" - title := "hcsshim::HNSEndpoint::" + operation - logrus.Debugf(title+" id=%s", endpoint.Id) - - requestMessage := &EndpointAttachDetachRequest{ - SystemType: VirtualMachineType, - } - response := &EndpointResquestResponse{} - - jsonString, err := json.Marshal(requestMessage) - if err != nil { - return err - } - return hnsCall("POST", "/endpoints/"+endpoint.Id+"/detach", string(jsonString), &response) -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hns/hnsfuncs.go b/vendor/github.com/Microsoft/hcsshim/internal/hns/hnsfuncs.go deleted file mode 100644 index 969d1b263..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/hns/hnsfuncs.go +++ /dev/null @@ -1,42 +0,0 @@ -package hns - -import ( - "encoding/json" - "fmt" - - "github.com/Microsoft/hcsshim/internal/hcserror" - "github.com/Microsoft/hcsshim/internal/interop" - "github.com/sirupsen/logrus" -) - -func hnsCall(method, path, request string, returnResponse interface{}) error { - var responseBuffer *uint16 - logrus.Debugf("[%s]=>[%s] Request : %s", method, path, request) - - err := _hnsCall(method, path, request, &responseBuffer) - if err != nil { - return hcserror.New(err, "hnsCall ", "") - } - response := interop.ConvertAndFreeCoTaskMemString(responseBuffer) - - hnsresponse := &hnsResponse{} - if err = json.Unmarshal([]byte(response), &hnsresponse); err != nil { - return err - } - - if !hnsresponse.Success { - return fmt.Errorf("HNS failed with error : %s", hnsresponse.Error) - } - - if len(hnsresponse.Output) == 0 { - return nil - } - - logrus.Debugf("Network Response : %s", hnsresponse.Output) - err = json.Unmarshal(hnsresponse.Output, returnResponse) - if err != nil { - return err - } - - return nil -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hns/hnsglobals.go b/vendor/github.com/Microsoft/hcsshim/internal/hns/hnsglobals.go deleted file mode 100644 index a8d8cc56a..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/hns/hnsglobals.go +++ /dev/null @@ -1,28 +0,0 @@ -package hns - -type HNSGlobals struct { - Version HNSVersion `json:"Version"` -} - -type HNSVersion struct { - Major int `json:"Major"` - Minor int `json:"Minor"` -} - -var ( - HNSVersion1803 = HNSVersion{Major: 7, Minor: 2} -) - -func GetHNSGlobals() (*HNSGlobals, error) { - var version HNSVersion - err := hnsCall("GET", "/globals/version", "", &version) - if err != nil { - return nil, err - } - - globals := &HNSGlobals{ - Version: version, - } - - return globals, nil -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hns/hnsnetwork.go b/vendor/github.com/Microsoft/hcsshim/internal/hns/hnsnetwork.go deleted file mode 100644 index 7e859de91..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/hns/hnsnetwork.go +++ /dev/null @@ -1,141 +0,0 @@ -package hns - -import ( - "encoding/json" - "net" - - "github.com/sirupsen/logrus" -) - -// Subnet is assoicated with a network and represents a list -// of subnets available to the network -type Subnet struct { - AddressPrefix string `json:",omitempty"` - GatewayAddress string `json:",omitempty"` - Policies []json.RawMessage `json:",omitempty"` -} - -// MacPool is assoicated with a network and represents a list -// of macaddresses available to the network -type MacPool struct { - StartMacAddress string `json:",omitempty"` - EndMacAddress string `json:",omitempty"` -} - -// HNSNetwork represents a network in HNS -type HNSNetwork struct { - Id string `json:"ID,omitempty"` - Name string `json:",omitempty"` - Type string `json:",omitempty"` - NetworkAdapterName string `json:",omitempty"` - SourceMac string `json:",omitempty"` - Policies []json.RawMessage `json:",omitempty"` - MacPools []MacPool `json:",omitempty"` - Subnets []Subnet `json:",omitempty"` - DNSSuffix string `json:",omitempty"` - DNSServerList string `json:",omitempty"` - DNSServerCompartment uint32 `json:",omitempty"` - ManagementIP string `json:",omitempty"` - AutomaticDNS bool `json:",omitempty"` -} - -type hnsNetworkResponse struct { - Success bool - Error string - Output HNSNetwork -} - -type hnsResponse struct { - Success bool - Error string - Output json.RawMessage -} - -// HNSNetworkRequest makes a call into HNS to update/query a single network -func HNSNetworkRequest(method, path, request string) (*HNSNetwork, error) { - var network HNSNetwork - err := hnsCall(method, "/networks/"+path, request, &network) - if err != nil { - return nil, err - } - - return &network, nil -} - -// HNSListNetworkRequest makes a HNS call to query the list of available networks -func HNSListNetworkRequest(method, path, request string) ([]HNSNetwork, error) { - var network []HNSNetwork - err := hnsCall(method, "/networks/"+path, request, &network) - if err != nil { - return nil, err - } - - return network, nil -} - -// GetHNSNetworkByID -func GetHNSNetworkByID(networkID string) (*HNSNetwork, error) { - return HNSNetworkRequest("GET", networkID, "") -} - -// GetHNSNetworkName filtered by Name -func GetHNSNetworkByName(networkName string) (*HNSNetwork, error) { - hsnnetworks, err := HNSListNetworkRequest("GET", "", "") - if err != nil { - return nil, err - } - for _, hnsnetwork := range hsnnetworks { - if hnsnetwork.Name == networkName { - return &hnsnetwork, nil - } - } - return nil, NetworkNotFoundError{NetworkName: networkName} -} - -// Create Network by sending NetworkRequest to HNS. -func (network *HNSNetwork) Create() (*HNSNetwork, error) { - operation := "Create" - title := "hcsshim::HNSNetwork::" + operation - logrus.Debugf(title+" id=%s", network.Id) - - jsonString, err := json.Marshal(network) - if err != nil { - return nil, err - } - return HNSNetworkRequest("POST", "", string(jsonString)) -} - -// Delete Network by sending NetworkRequest to HNS -func (network *HNSNetwork) Delete() (*HNSNetwork, error) { - operation := "Delete" - title := "hcsshim::HNSNetwork::" + operation - logrus.Debugf(title+" id=%s", network.Id) - - return HNSNetworkRequest("DELETE", network.Id, "") -} - -// Creates an endpoint on the Network. -func (network *HNSNetwork) NewEndpoint(ipAddress net.IP, macAddress net.HardwareAddr) *HNSEndpoint { - return &HNSEndpoint{ - VirtualNetwork: network.Id, - IPAddress: ipAddress, - MacAddress: string(macAddress), - } -} - -func (network *HNSNetwork) CreateEndpoint(endpoint *HNSEndpoint) (*HNSEndpoint, error) { - operation := "CreateEndpoint" - title := "hcsshim::HNSNetwork::" + operation - logrus.Debugf(title+" id=%s, endpointId=%s", network.Id, endpoint.Id) - - endpoint.VirtualNetwork = network.Id - return endpoint.Create() -} - -func (network *HNSNetwork) CreateRemoteEndpoint(endpoint *HNSEndpoint) (*HNSEndpoint, error) { - operation := "CreateRemoteEndpoint" - title := "hcsshim::HNSNetwork::" + operation - logrus.Debugf(title+" id=%s", network.Id) - endpoint.IsRemoteEndpoint = true - return network.CreateEndpoint(endpoint) -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hns/hnspolicy.go b/vendor/github.com/Microsoft/hcsshim/internal/hns/hnspolicy.go deleted file mode 100644 index 2318a4fce..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/hns/hnspolicy.go +++ /dev/null @@ -1,98 +0,0 @@ -package hns - -// Type of Request Support in ModifySystem -type PolicyType string - -// RequestType const -const ( - Nat PolicyType = "NAT" - ACL PolicyType = "ACL" - PA PolicyType = "PA" - VLAN PolicyType = "VLAN" - VSID PolicyType = "VSID" - VNet PolicyType = "VNET" - L2Driver PolicyType = "L2Driver" - Isolation PolicyType = "Isolation" - QOS PolicyType = "QOS" - OutboundNat PolicyType = "OutBoundNAT" - ExternalLoadBalancer PolicyType = "ELB" - Route PolicyType = "ROUTE" -) - -type NatPolicy struct { - Type PolicyType `json:"Type"` - Protocol string - InternalPort uint16 - ExternalPort uint16 -} - -type QosPolicy struct { - Type PolicyType `json:"Type"` - MaximumOutgoingBandwidthInBytes uint64 -} - -type IsolationPolicy struct { - Type PolicyType `json:"Type"` - VLAN uint - VSID uint - InDefaultIsolation bool -} - -type VlanPolicy struct { - Type PolicyType `json:"Type"` - VLAN uint -} - -type VsidPolicy struct { - Type PolicyType `json:"Type"` - VSID uint -} - -type PaPolicy struct { - Type PolicyType `json:"Type"` - PA string `json:"PA"` -} - -type OutboundNatPolicy struct { - Policy - VIP string `json:"VIP,omitempty"` - Exceptions []string `json:"ExceptionList,omitempty"` -} - -type ActionType string -type DirectionType string -type RuleType string - -const ( - Allow ActionType = "Allow" - Block ActionType = "Block" - - In DirectionType = "In" - Out DirectionType = "Out" - - Host RuleType = "Host" - Switch RuleType = "Switch" -) - -type ACLPolicy struct { - Type PolicyType `json:"Type"` - Id string `json:"Id,omitempty"` - Protocol uint16 - Protocols string `json:"Protocols,omitempty"` - InternalPort uint16 - Action ActionType - Direction DirectionType - LocalAddresses string - RemoteAddresses string - LocalPorts string `json:"LocalPorts,omitempty"` - LocalPort uint16 - RemotePorts string `json:"RemotePorts,omitempty"` - RemotePort uint16 - RuleType RuleType `json:"RuleType,omitempty"` - Priority uint16 - ServiceName string -} - -type Policy struct { - Type PolicyType `json:"Type"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hns/hnspolicylist.go b/vendor/github.com/Microsoft/hcsshim/internal/hns/hnspolicylist.go deleted file mode 100644 index 31322a681..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/hns/hnspolicylist.go +++ /dev/null @@ -1,201 +0,0 @@ -package hns - -import ( - "encoding/json" - - "github.com/sirupsen/logrus" -) - -// RoutePolicy is a structure defining schema for Route based Policy -type RoutePolicy struct { - Policy - DestinationPrefix string `json:"DestinationPrefix,omitempty"` - NextHop string `json:"NextHop,omitempty"` - EncapEnabled bool `json:"NeedEncap,omitempty"` -} - -// ELBPolicy is a structure defining schema for ELB LoadBalancing based Policy -type ELBPolicy struct { - LBPolicy - SourceVIP string `json:"SourceVIP,omitempty"` - VIPs []string `json:"VIPs,omitempty"` - ILB bool `json:"ILB,omitempty"` - DSR bool `json:"IsDSR,omitempty"` -} - -// LBPolicy is a structure defining schema for LoadBalancing based Policy -type LBPolicy struct { - Policy - Protocol uint16 `json:"Protocol,omitempty"` - InternalPort uint16 - ExternalPort uint16 -} - -// PolicyList is a structure defining schema for Policy list request -type PolicyList struct { - ID string `json:"ID,omitempty"` - EndpointReferences []string `json:"References,omitempty"` - Policies []json.RawMessage `json:"Policies,omitempty"` -} - -// HNSPolicyListRequest makes a call into HNS to update/query a single network -func HNSPolicyListRequest(method, path, request string) (*PolicyList, error) { - var policy PolicyList - err := hnsCall(method, "/policylists/"+path, request, &policy) - if err != nil { - return nil, err - } - - return &policy, nil -} - -// HNSListPolicyListRequest gets all the policy list -func HNSListPolicyListRequest() ([]PolicyList, error) { - var plist []PolicyList - err := hnsCall("GET", "/policylists/", "", &plist) - if err != nil { - return nil, err - } - - return plist, nil -} - -// PolicyListRequest makes a HNS call to modify/query a network policy list -func PolicyListRequest(method, path, request string) (*PolicyList, error) { - policylist := &PolicyList{} - err := hnsCall(method, "/policylists/"+path, request, &policylist) - if err != nil { - return nil, err - } - - return policylist, nil -} - -// GetPolicyListByID get the policy list by ID -func GetPolicyListByID(policyListID string) (*PolicyList, error) { - return PolicyListRequest("GET", policyListID, "") -} - -// Create PolicyList by sending PolicyListRequest to HNS. -func (policylist *PolicyList) Create() (*PolicyList, error) { - operation := "Create" - title := "hcsshim::PolicyList::" + operation - logrus.Debugf(title+" id=%s", policylist.ID) - jsonString, err := json.Marshal(policylist) - if err != nil { - return nil, err - } - return PolicyListRequest("POST", "", string(jsonString)) -} - -// Delete deletes PolicyList -func (policylist *PolicyList) Delete() (*PolicyList, error) { - operation := "Delete" - title := "hcsshim::PolicyList::" + operation - logrus.Debugf(title+" id=%s", policylist.ID) - - return PolicyListRequest("DELETE", policylist.ID, "") -} - -// AddEndpoint add an endpoint to a Policy List -func (policylist *PolicyList) AddEndpoint(endpoint *HNSEndpoint) (*PolicyList, error) { - operation := "AddEndpoint" - title := "hcsshim::PolicyList::" + operation - logrus.Debugf(title+" id=%s, endpointId:%s", policylist.ID, endpoint.Id) - - _, err := policylist.Delete() - if err != nil { - return nil, err - } - - // Add Endpoint to the Existing List - policylist.EndpointReferences = append(policylist.EndpointReferences, "/endpoints/"+endpoint.Id) - - return policylist.Create() -} - -// RemoveEndpoint removes an endpoint from the Policy List -func (policylist *PolicyList) RemoveEndpoint(endpoint *HNSEndpoint) (*PolicyList, error) { - operation := "RemoveEndpoint" - title := "hcsshim::PolicyList::" + operation - logrus.Debugf(title+" id=%s, endpointId:%s", policylist.ID, endpoint.Id) - - _, err := policylist.Delete() - if err != nil { - return nil, err - } - - elementToRemove := "/endpoints/" + endpoint.Id - - var references []string - - for _, endpointReference := range policylist.EndpointReferences { - if endpointReference == elementToRemove { - continue - } - references = append(references, endpointReference) - } - policylist.EndpointReferences = references - return policylist.Create() -} - -// AddLoadBalancer policy list for the specified endpoints -func AddLoadBalancer(endpoints []HNSEndpoint, isILB bool, sourceVIP, vip string, protocol uint16, internalPort uint16, externalPort uint16) (*PolicyList, error) { - operation := "AddLoadBalancer" - title := "hcsshim::PolicyList::" + operation - logrus.Debugf(title+" endpointId=%v, isILB=%v, sourceVIP=%s, vip=%s, protocol=%v, internalPort=%v, externalPort=%v", endpoints, isILB, sourceVIP, vip, protocol, internalPort, externalPort) - - policylist := &PolicyList{} - - elbPolicy := &ELBPolicy{ - SourceVIP: sourceVIP, - ILB: isILB, - } - - if len(vip) > 0 { - elbPolicy.VIPs = []string{vip} - } - elbPolicy.Type = ExternalLoadBalancer - elbPolicy.Protocol = protocol - elbPolicy.InternalPort = internalPort - elbPolicy.ExternalPort = externalPort - - for _, endpoint := range endpoints { - policylist.EndpointReferences = append(policylist.EndpointReferences, "/endpoints/"+endpoint.Id) - } - - jsonString, err := json.Marshal(elbPolicy) - if err != nil { - return nil, err - } - policylist.Policies = append(policylist.Policies, jsonString) - return policylist.Create() -} - -// AddRoute adds route policy list for the specified endpoints -func AddRoute(endpoints []HNSEndpoint, destinationPrefix string, nextHop string, encapEnabled bool) (*PolicyList, error) { - operation := "AddRoute" - title := "hcsshim::PolicyList::" + operation - logrus.Debugf(title+" destinationPrefix:%s", destinationPrefix) - - policylist := &PolicyList{} - - rPolicy := &RoutePolicy{ - DestinationPrefix: destinationPrefix, - NextHop: nextHop, - EncapEnabled: encapEnabled, - } - rPolicy.Type = Route - - for _, endpoint := range endpoints { - policylist.EndpointReferences = append(policylist.EndpointReferences, "/endpoints/"+endpoint.Id) - } - - jsonString, err := json.Marshal(rPolicy) - if err != nil { - return nil, err - } - - policylist.Policies = append(policylist.Policies, jsonString) - return policylist.Create() -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hns/hnssupport.go b/vendor/github.com/Microsoft/hcsshim/internal/hns/hnssupport.go deleted file mode 100644 index d5efba7f2..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/hns/hnssupport.go +++ /dev/null @@ -1,49 +0,0 @@ -package hns - -import ( - "github.com/sirupsen/logrus" -) - -type HNSSupportedFeatures struct { - Acl HNSAclFeatures `json:"ACL"` -} - -type HNSAclFeatures struct { - AclAddressLists bool `json:"AclAddressLists"` - AclNoHostRulePriority bool `json:"AclHostRulePriority"` - AclPortRanges bool `json:"AclPortRanges"` - AclRuleId bool `json:"AclRuleId"` -} - -func GetHNSSupportedFeatures() HNSSupportedFeatures { - var hnsFeatures HNSSupportedFeatures - - globals, err := GetHNSGlobals() - if err != nil { - // Expected on pre-1803 builds, all features will be false/unsupported - logrus.Debugf("Unable to obtain HNS globals: %s", err) - return hnsFeatures - } - - hnsFeatures.Acl = HNSAclFeatures{ - AclAddressLists: isHNSFeatureSupported(globals.Version, HNSVersion1803), - AclNoHostRulePriority: isHNSFeatureSupported(globals.Version, HNSVersion1803), - AclPortRanges: isHNSFeatureSupported(globals.Version, HNSVersion1803), - AclRuleId: isHNSFeatureSupported(globals.Version, HNSVersion1803), - } - - return hnsFeatures -} - -func isHNSFeatureSupported(currentVersion HNSVersion, minVersionSupported HNSVersion) bool { - if currentVersion.Major < minVersionSupported.Major { - return false - } - if currentVersion.Major > minVersionSupported.Major { - return true - } - if currentVersion.Minor < minVersionSupported.Minor { - return false - } - return true -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hns/namespace.go b/vendor/github.com/Microsoft/hcsshim/internal/hns/namespace.go deleted file mode 100644 index 45e2281b0..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/hns/namespace.go +++ /dev/null @@ -1,110 +0,0 @@ -package hns - -import ( - "encoding/json" - "fmt" - "os" - "path" - "strings" -) - -type namespaceRequest struct { - IsDefault bool `json:",omitempty"` -} - -type namespaceEndpointRequest struct { - ID string `json:"Id"` -} - -type NamespaceResource struct { - Type string - Data json.RawMessage -} - -type namespaceResourceRequest struct { - Type string - Data interface{} -} - -type Namespace struct { - ID string - IsDefault bool `json:",omitempty"` - ResourceList []NamespaceResource `json:",omitempty"` -} - -func issueNamespaceRequest(id *string, method, subpath string, request interface{}) (*Namespace, error) { - var err error - hnspath := "/namespaces/" - if id != nil { - hnspath = path.Join(hnspath, *id) - } - if subpath != "" { - hnspath = path.Join(hnspath, subpath) - } - var reqJSON []byte - if request != nil { - if reqJSON, err = json.Marshal(request); err != nil { - return nil, err - } - } - var ns Namespace - err = hnsCall(method, hnspath, string(reqJSON), &ns) - if err != nil { - if strings.Contains(err.Error(), "Element not found.") { - return nil, os.ErrNotExist - } - return nil, fmt.Errorf("%s %s: %s", method, hnspath, err) - } - return &ns, err -} - -func CreateNamespace() (string, error) { - req := namespaceRequest{} - ns, err := issueNamespaceRequest(nil, "POST", "", &req) - if err != nil { - return "", err - } - return ns.ID, nil -} - -func RemoveNamespace(id string) error { - _, err := issueNamespaceRequest(&id, "DELETE", "", nil) - return err -} - -func GetNamespaceEndpoints(id string) ([]string, error) { - ns, err := issueNamespaceRequest(&id, "GET", "", nil) - if err != nil { - return nil, err - } - var endpoints []string - for _, rsrc := range ns.ResourceList { - if rsrc.Type == "Endpoint" { - var endpoint namespaceEndpointRequest - err = json.Unmarshal(rsrc.Data, &endpoint) - if err != nil { - return nil, fmt.Errorf("unmarshal endpoint: %s", err) - } - endpoints = append(endpoints, endpoint.ID) - } - } - return endpoints, nil -} - -func AddNamespaceEndpoint(id string, endpointID string) error { - resource := namespaceResourceRequest{ - Type: "Endpoint", - Data: namespaceEndpointRequest{endpointID}, - } - _, err := issueNamespaceRequest(&id, "POST", "addresource", &resource) - return err -} - -func RemoveNamespaceEndpoint(id string, endpointID string) error { - resource := namespaceResourceRequest{ - Type: "Endpoint", - Data: namespaceEndpointRequest{endpointID}, - } - _, err := issueNamespaceRequest(&id, "POST", "removeresource", &resource) - return err -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hns/zsyscall_windows.go b/vendor/github.com/Microsoft/hcsshim/internal/hns/zsyscall_windows.go deleted file mode 100644 index 204633a48..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/hns/zsyscall_windows.go +++ /dev/null @@ -1,76 +0,0 @@ -// Code generated mksyscall_windows.exe DO NOT EDIT - -package hns - -import ( - "syscall" - "unsafe" - - "golang.org/x/sys/windows" -) - -var _ unsafe.Pointer - -// Do the interface allocations only once for common -// Errno values. -const ( - errnoERROR_IO_PENDING = 997 -) - -var ( - errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING) -) - -// errnoErr returns common boxed Errno values, to prevent -// allocations at runtime. -func errnoErr(e syscall.Errno) error { - switch e { - case 0: - return nil - case errnoERROR_IO_PENDING: - return errERROR_IO_PENDING - } - // TODO: add more here, after collecting data on the common - // error values see on Windows. (perhaps when running - // all.bat?) - return e -} - -var ( - modvmcompute = windows.NewLazySystemDLL("vmcompute.dll") - - procHNSCall = modvmcompute.NewProc("HNSCall") -) - -func _hnsCall(method string, path string, object string, response **uint16) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(method) - if hr != nil { - return - } - var _p1 *uint16 - _p1, hr = syscall.UTF16PtrFromString(path) - if hr != nil { - return - } - var _p2 *uint16 - _p2, hr = syscall.UTF16PtrFromString(object) - if hr != nil { - return - } - return __hnsCall(_p0, _p1, _p2, response) -} - -func __hnsCall(method *uint16, path *uint16, object *uint16, response **uint16) (hr error) { - if hr = procHNSCall.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall6(procHNSCall.Addr(), 4, uintptr(unsafe.Pointer(method)), uintptr(unsafe.Pointer(path)), uintptr(unsafe.Pointer(object)), uintptr(unsafe.Pointer(response)), 0, 0) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/interop/interop.go b/vendor/github.com/Microsoft/hcsshim/internal/interop/interop.go deleted file mode 100644 index 2f6ec029e..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/interop/interop.go +++ /dev/null @@ -1,27 +0,0 @@ -package interop - -import ( - "syscall" - "unsafe" -) - -//go:generate go run ../../mksyscall_windows.go -output zsyscall_windows.go interop.go - -//sys coTaskMemFree(buffer unsafe.Pointer) = api_ms_win_core_com_l1_1_0.CoTaskMemFree - -func ConvertAndFreeCoTaskMemString(buffer *uint16) string { - str := syscall.UTF16ToString((*[1 << 29]uint16)(unsafe.Pointer(buffer))[:]) - coTaskMemFree(unsafe.Pointer(buffer)) - return str -} - -func ConvertAndFreeCoTaskMemBytes(buffer *uint16) []byte { - return []byte(ConvertAndFreeCoTaskMemString(buffer)) -} - -func Win32FromHresult(hr uintptr) syscall.Errno { - if hr&0x1fff0000 == 0x00070000 { - return syscall.Errno(hr & 0xffff) - } - return syscall.Errno(hr) -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/interop/zsyscall_windows.go b/vendor/github.com/Microsoft/hcsshim/internal/interop/zsyscall_windows.go deleted file mode 100644 index 12b0c71c5..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/interop/zsyscall_windows.go +++ /dev/null @@ -1,48 +0,0 @@ -// Code generated mksyscall_windows.exe DO NOT EDIT - -package interop - -import ( - "syscall" - "unsafe" - - "golang.org/x/sys/windows" -) - -var _ unsafe.Pointer - -// Do the interface allocations only once for common -// Errno values. -const ( - errnoERROR_IO_PENDING = 997 -) - -var ( - errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING) -) - -// errnoErr returns common boxed Errno values, to prevent -// allocations at runtime. -func errnoErr(e syscall.Errno) error { - switch e { - case 0: - return nil - case errnoERROR_IO_PENDING: - return errERROR_IO_PENDING - } - // TODO: add more here, after collecting data on the common - // error values see on Windows. (perhaps when running - // all.bat?) - return e -} - -var ( - modapi_ms_win_core_com_l1_1_0 = windows.NewLazySystemDLL("api-ms-win-core-com-l1-1-0.dll") - - procCoTaskMemFree = modapi_ms_win_core_com_l1_1_0.NewProc("CoTaskMemFree") -) - -func coTaskMemFree(buffer unsafe.Pointer) { - syscall.Syscall(procCoTaskMemFree.Addr(), 1, uintptr(buffer), 0, 0) - return -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/logfields/fields.go b/vendor/github.com/Microsoft/hcsshim/internal/logfields/fields.go deleted file mode 100644 index cf2c166d9..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/logfields/fields.go +++ /dev/null @@ -1,32 +0,0 @@ -package logfields - -const ( - // Identifiers - - ContainerID = "cid" - UVMID = "uvm-id" - ProcessID = "pid" - - // Common Misc - - // Timeout represents an operation timeout. - Timeout = "timeout" - JSON = "json" - - // Keys/values - - Field = "field" - OCIAnnotation = "oci-annotation" - Value = "value" - - // Golang type's - - ExpectedType = "expected-type" - Bool = "bool" - Uint32 = "uint32" - Uint64 = "uint64" - - // runhcs - - VMShimOperation = "vmshim-op" -) diff --git a/vendor/github.com/Microsoft/hcsshim/internal/longpath/longpath.go b/vendor/github.com/Microsoft/hcsshim/internal/longpath/longpath.go deleted file mode 100644 index e5b8b85e0..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/longpath/longpath.go +++ /dev/null @@ -1,24 +0,0 @@ -package longpath - -import ( - "path/filepath" - "strings" -) - -// LongAbs makes a path absolute and returns it in NT long path form. -func LongAbs(path string) (string, error) { - if strings.HasPrefix(path, `\\?\`) || strings.HasPrefix(path, `\\.\`) { - return path, nil - } - if !filepath.IsAbs(path) { - absPath, err := filepath.Abs(path) - if err != nil { - return "", err - } - path = absPath - } - if strings.HasPrefix(path, `\\`) { - return `\\?\UNC\` + path[2:], nil - } - return `\\?\` + path, nil -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/mergemaps/merge.go b/vendor/github.com/Microsoft/hcsshim/internal/mergemaps/merge.go deleted file mode 100644 index 7e95efb30..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/mergemaps/merge.go +++ /dev/null @@ -1,52 +0,0 @@ -package mergemaps - -import "encoding/json" - -// Merge recursively merges map `fromMap` into map `ToMap`. Any pre-existing values -// in ToMap are overwritten. Values in fromMap are added to ToMap. -// From http://stackoverflow.com/questions/40491438/merging-two-json-strings-in-golang -func Merge(fromMap, ToMap interface{}) interface{} { - switch fromMap := fromMap.(type) { - case map[string]interface{}: - ToMap, ok := ToMap.(map[string]interface{}) - if !ok { - return fromMap - } - for keyToMap, valueToMap := range ToMap { - if valueFromMap, ok := fromMap[keyToMap]; ok { - fromMap[keyToMap] = Merge(valueFromMap, valueToMap) - } else { - fromMap[keyToMap] = valueToMap - } - } - case nil: - // merge(nil, map[string]interface{...}) -> map[string]interface{...} - ToMap, ok := ToMap.(map[string]interface{}) - if ok { - return ToMap - } - } - return fromMap -} - -// MergeJSON merges the contents of a JSON string into an object representation, -// returning a new object suitable for translating to JSON. -func MergeJSON(object interface{}, additionalJSON []byte) (interface{}, error) { - if len(additionalJSON) == 0 { - return object, nil - } - objectJSON, err := json.Marshal(object) - if err != nil { - return nil, err - } - var objectMap, newMap map[string]interface{} - err = json.Unmarshal(objectJSON, &objectMap) - if err != nil { - return nil, err - } - err = json.Unmarshal(additionalJSON, &newMap) - if err != nil { - return nil, err - } - return Merge(newMap, objectMap), nil -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/safefile/safeopen.go b/vendor/github.com/Microsoft/hcsshim/internal/safefile/safeopen.go deleted file mode 100644 index f31edfaf8..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/safefile/safeopen.go +++ /dev/null @@ -1,431 +0,0 @@ -package safefile - -import ( - "errors" - "io" - "os" - "path/filepath" - "strings" - "syscall" - "unicode/utf16" - "unsafe" - - "github.com/Microsoft/hcsshim/internal/longpath" - - winio "github.com/Microsoft/go-winio" -) - -//go:generate go run $GOROOT\src\syscall\mksyscall_windows.go -output zsyscall_windows.go safeopen.go - -//sys ntCreateFile(handle *uintptr, accessMask uint32, oa *objectAttributes, iosb *ioStatusBlock, allocationSize *uint64, fileAttributes uint32, shareAccess uint32, createDisposition uint32, createOptions uint32, eaBuffer *byte, eaLength uint32) (status uint32) = ntdll.NtCreateFile -//sys ntSetInformationFile(handle uintptr, iosb *ioStatusBlock, information uintptr, length uint32, class uint32) (status uint32) = ntdll.NtSetInformationFile -//sys rtlNtStatusToDosError(status uint32) (winerr error) = ntdll.RtlNtStatusToDosErrorNoTeb -//sys localAlloc(flags uint32, size int) (ptr uintptr) = kernel32.LocalAlloc -//sys localFree(ptr uintptr) = kernel32.LocalFree - -type ioStatusBlock struct { - Status, Information uintptr -} - -type objectAttributes struct { - Length uintptr - RootDirectory uintptr - ObjectName uintptr - Attributes uintptr - SecurityDescriptor uintptr - SecurityQoS uintptr -} - -type unicodeString struct { - Length uint16 - MaximumLength uint16 - Buffer uintptr -} - -type fileLinkInformation struct { - ReplaceIfExists bool - RootDirectory uintptr - FileNameLength uint32 - FileName [1]uint16 -} - -type fileDispositionInformationEx struct { - Flags uintptr -} - -const ( - _FileLinkInformation = 11 - _FileDispositionInformationEx = 64 - - FILE_READ_ATTRIBUTES = 0x0080 - FILE_WRITE_ATTRIBUTES = 0x0100 - DELETE = 0x10000 - - FILE_OPEN = 1 - FILE_CREATE = 2 - - FILE_DIRECTORY_FILE = 0x00000001 - FILE_SYNCHRONOUS_IO_NONALERT = 0x00000020 - FILE_DELETE_ON_CLOSE = 0x00001000 - FILE_OPEN_FOR_BACKUP_INTENT = 0x00004000 - FILE_OPEN_REPARSE_POINT = 0x00200000 - - FILE_DISPOSITION_DELETE = 0x00000001 - - _OBJ_DONT_REPARSE = 0x1000 - - _STATUS_REPARSE_POINT_ENCOUNTERED = 0xC000050B -) - -func OpenRoot(path string) (*os.File, error) { - longpath, err := longpath.LongAbs(path) - if err != nil { - return nil, err - } - return winio.OpenForBackup(longpath, syscall.GENERIC_READ, syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, syscall.OPEN_EXISTING) -} - -func ntRelativePath(path string) ([]uint16, error) { - path = filepath.Clean(path) - if strings.Contains(path, ":") { - // Since alternate data streams must follow the file they - // are attached to, finding one here (out of order) is invalid. - return nil, errors.New("path contains invalid character `:`") - } - fspath := filepath.FromSlash(path) - if len(fspath) > 0 && fspath[0] == '\\' { - return nil, errors.New("expected relative path") - } - - path16 := utf16.Encode(([]rune)(fspath)) - if len(path16) > 32767 { - return nil, syscall.ENAMETOOLONG - } - - return path16, nil -} - -// openRelativeInternal opens a relative path from the given root, failing if -// any of the intermediate path components are reparse points. -func openRelativeInternal(path string, root *os.File, accessMask uint32, shareFlags uint32, createDisposition uint32, flags uint32) (*os.File, error) { - var ( - h uintptr - iosb ioStatusBlock - oa objectAttributes - ) - - path16, err := ntRelativePath(path) - if err != nil { - return nil, err - } - - if root == nil || root.Fd() == 0 { - return nil, errors.New("missing root directory") - } - - upathBuffer := localAlloc(0, int(unsafe.Sizeof(unicodeString{}))+len(path16)*2) - defer localFree(upathBuffer) - - upath := (*unicodeString)(unsafe.Pointer(upathBuffer)) - upath.Length = uint16(len(path16) * 2) - upath.MaximumLength = upath.Length - upath.Buffer = upathBuffer + unsafe.Sizeof(*upath) - copy((*[32768]uint16)(unsafe.Pointer(upath.Buffer))[:], path16) - - oa.Length = unsafe.Sizeof(oa) - oa.ObjectName = upathBuffer - oa.RootDirectory = uintptr(root.Fd()) - oa.Attributes = _OBJ_DONT_REPARSE - status := ntCreateFile( - &h, - accessMask|syscall.SYNCHRONIZE, - &oa, - &iosb, - nil, - 0, - shareFlags, - createDisposition, - FILE_OPEN_FOR_BACKUP_INTENT|FILE_SYNCHRONOUS_IO_NONALERT|flags, - nil, - 0, - ) - if status != 0 { - return nil, rtlNtStatusToDosError(status) - } - - fullPath, err := longpath.LongAbs(filepath.Join(root.Name(), path)) - if err != nil { - syscall.Close(syscall.Handle(h)) - return nil, err - } - - return os.NewFile(h, fullPath), nil -} - -// OpenRelative opens a relative path from the given root, failing if -// any of the intermediate path components are reparse points. -func OpenRelative(path string, root *os.File, accessMask uint32, shareFlags uint32, createDisposition uint32, flags uint32) (*os.File, error) { - f, err := openRelativeInternal(path, root, accessMask, shareFlags, createDisposition, flags) - if err != nil { - err = &os.PathError{Op: "open", Path: filepath.Join(root.Name(), path), Err: err} - } - return f, err -} - -// LinkRelative creates a hard link from oldname to newname (relative to oldroot -// and newroot), failing if any of the intermediate path components are reparse -// points. -func LinkRelative(oldname string, oldroot *os.File, newname string, newroot *os.File) error { - // Open the old file. - oldf, err := openRelativeInternal( - oldname, - oldroot, - syscall.FILE_WRITE_ATTRIBUTES, - syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, - FILE_OPEN, - 0, - ) - if err != nil { - return &os.LinkError{Op: "link", Old: filepath.Join(oldroot.Name(), oldname), New: filepath.Join(newroot.Name(), newname), Err: err} - } - defer oldf.Close() - - // Open the parent of the new file. - var parent *os.File - parentPath := filepath.Dir(newname) - if parentPath != "." { - parent, err = openRelativeInternal( - parentPath, - newroot, - syscall.GENERIC_READ, - syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, - FILE_OPEN, - FILE_DIRECTORY_FILE) - if err != nil { - return &os.LinkError{Op: "link", Old: oldf.Name(), New: filepath.Join(newroot.Name(), newname), Err: err} - } - defer parent.Close() - - fi, err := winio.GetFileBasicInfo(parent) - if err != nil { - return err - } - if (fi.FileAttributes & syscall.FILE_ATTRIBUTE_REPARSE_POINT) != 0 { - return &os.LinkError{Op: "link", Old: oldf.Name(), New: filepath.Join(newroot.Name(), newname), Err: rtlNtStatusToDosError(_STATUS_REPARSE_POINT_ENCOUNTERED)} - } - - } else { - parent = newroot - } - - // Issue an NT call to create the link. This will be safe because NT will - // not open any more directories to create the link, so it cannot walk any - // more reparse points. - newbase := filepath.Base(newname) - newbase16, err := ntRelativePath(newbase) - if err != nil { - return err - } - - size := int(unsafe.Offsetof(fileLinkInformation{}.FileName)) + len(newbase16)*2 - linkinfoBuffer := localAlloc(0, size) - defer localFree(linkinfoBuffer) - linkinfo := (*fileLinkInformation)(unsafe.Pointer(linkinfoBuffer)) - linkinfo.RootDirectory = parent.Fd() - linkinfo.FileNameLength = uint32(len(newbase16) * 2) - copy((*[32768]uint16)(unsafe.Pointer(&linkinfo.FileName[0]))[:], newbase16) - - var iosb ioStatusBlock - status := ntSetInformationFile( - oldf.Fd(), - &iosb, - linkinfoBuffer, - uint32(size), - _FileLinkInformation, - ) - if status != 0 { - return &os.LinkError{Op: "link", Old: oldf.Name(), New: filepath.Join(parent.Name(), newbase), Err: rtlNtStatusToDosError(status)} - } - - return nil -} - -// deleteOnClose marks a file to be deleted when the handle is closed. -func deleteOnClose(f *os.File) error { - disposition := fileDispositionInformationEx{Flags: FILE_DISPOSITION_DELETE} - var iosb ioStatusBlock - status := ntSetInformationFile( - f.Fd(), - &iosb, - uintptr(unsafe.Pointer(&disposition)), - uint32(unsafe.Sizeof(disposition)), - _FileDispositionInformationEx, - ) - if status != 0 { - return rtlNtStatusToDosError(status) - } - return nil -} - -// clearReadOnly clears the readonly attribute on a file. -func clearReadOnly(f *os.File) error { - bi, err := winio.GetFileBasicInfo(f) - if err != nil { - return err - } - if bi.FileAttributes&syscall.FILE_ATTRIBUTE_READONLY == 0 { - return nil - } - sbi := winio.FileBasicInfo{ - FileAttributes: bi.FileAttributes &^ syscall.FILE_ATTRIBUTE_READONLY, - } - if sbi.FileAttributes == 0 { - sbi.FileAttributes = syscall.FILE_ATTRIBUTE_NORMAL - } - return winio.SetFileBasicInfo(f, &sbi) -} - -// RemoveRelative removes a file or directory relative to a root, failing if any -// intermediate path components are reparse points. -func RemoveRelative(path string, root *os.File) error { - f, err := openRelativeInternal( - path, - root, - FILE_READ_ATTRIBUTES|FILE_WRITE_ATTRIBUTES|DELETE, - syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, - FILE_OPEN, - FILE_OPEN_REPARSE_POINT) - if err == nil { - defer f.Close() - err = deleteOnClose(f) - if err == syscall.ERROR_ACCESS_DENIED { - // Maybe the file is marked readonly. Clear the bit and retry. - clearReadOnly(f) - err = deleteOnClose(f) - } - } - if err != nil { - return &os.PathError{Op: "remove", Path: filepath.Join(root.Name(), path), Err: err} - } - return nil -} - -// RemoveAllRelative removes a directory tree relative to a root, failing if any -// intermediate path components are reparse points. -func RemoveAllRelative(path string, root *os.File) error { - fi, err := LstatRelative(path, root) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - fileAttributes := fi.Sys().(*syscall.Win32FileAttributeData).FileAttributes - if fileAttributes&syscall.FILE_ATTRIBUTE_DIRECTORY == 0 || fileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT != 0 { - // If this is a reparse point, it can't have children. Simple remove will do. - err := RemoveRelative(path, root) - if err == nil || os.IsNotExist(err) { - return nil - } - return err - } - - // It is necessary to use os.Open as Readdirnames does not work with - // OpenRelative. This is safe because the above lstatrelative fails - // if the target is outside the root, and we know this is not a - // symlink from the above FILE_ATTRIBUTE_REPARSE_POINT check. - fd, err := os.Open(filepath.Join(root.Name(), path)) - if err != nil { - if os.IsNotExist(err) { - // Race. It was deleted between the Lstat and Open. - // Return nil per RemoveAll's docs. - return nil - } - return err - } - - // Remove contents & return first error. - for { - names, err1 := fd.Readdirnames(100) - for _, name := range names { - err1 := RemoveAllRelative(path+string(os.PathSeparator)+name, root) - if err == nil { - err = err1 - } - } - if err1 == io.EOF { - break - } - // If Readdirnames returned an error, use it. - if err == nil { - err = err1 - } - if len(names) == 0 { - break - } - } - fd.Close() - - // Remove directory. - err1 := RemoveRelative(path, root) - if err1 == nil || os.IsNotExist(err1) { - return nil - } - if err == nil { - err = err1 - } - return err -} - -// MkdirRelative creates a directory relative to a root, failing if any -// intermediate path components are reparse points. -func MkdirRelative(path string, root *os.File) error { - f, err := openRelativeInternal( - path, - root, - 0, - syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, - FILE_CREATE, - FILE_DIRECTORY_FILE) - if err == nil { - f.Close() - } else { - err = &os.PathError{Op: "mkdir", Path: filepath.Join(root.Name(), path), Err: err} - } - return err -} - -// LstatRelative performs a stat operation on a file relative to a root, failing -// if any intermediate path components are reparse points. -func LstatRelative(path string, root *os.File) (os.FileInfo, error) { - f, err := openRelativeInternal( - path, - root, - FILE_READ_ATTRIBUTES, - syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, - FILE_OPEN, - FILE_OPEN_REPARSE_POINT) - if err != nil { - return nil, &os.PathError{Op: "stat", Path: filepath.Join(root.Name(), path), Err: err} - } - defer f.Close() - return f.Stat() -} - -// EnsureNotReparsePointRelative validates that a given file (relative to a -// root) and all intermediate path components are not a reparse points. -func EnsureNotReparsePointRelative(path string, root *os.File) error { - // Perform an open with OBJ_DONT_REPARSE but without specifying FILE_OPEN_REPARSE_POINT. - f, err := OpenRelative( - path, - root, - 0, - syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, - FILE_OPEN, - 0) - if err != nil { - return err - } - f.Close() - return nil -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/safefile/zsyscall_windows.go b/vendor/github.com/Microsoft/hcsshim/internal/safefile/zsyscall_windows.go deleted file mode 100644 index 709b9d347..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/safefile/zsyscall_windows.go +++ /dev/null @@ -1,79 +0,0 @@ -// Code generated by 'go generate'; DO NOT EDIT. - -package safefile - -import ( - "syscall" - "unsafe" - - "golang.org/x/sys/windows" -) - -var _ unsafe.Pointer - -// Do the interface allocations only once for common -// Errno values. -const ( - errnoERROR_IO_PENDING = 997 -) - -var ( - errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING) -) - -// errnoErr returns common boxed Errno values, to prevent -// allocations at runtime. -func errnoErr(e syscall.Errno) error { - switch e { - case 0: - return nil - case errnoERROR_IO_PENDING: - return errERROR_IO_PENDING - } - // TODO: add more here, after collecting data on the common - // error values see on Windows. (perhaps when running - // all.bat?) - return e -} - -var ( - modntdll = windows.NewLazySystemDLL("ntdll.dll") - modkernel32 = windows.NewLazySystemDLL("kernel32.dll") - - procNtCreateFile = modntdll.NewProc("NtCreateFile") - procNtSetInformationFile = modntdll.NewProc("NtSetInformationFile") - procRtlNtStatusToDosErrorNoTeb = modntdll.NewProc("RtlNtStatusToDosErrorNoTeb") - procLocalAlloc = modkernel32.NewProc("LocalAlloc") - procLocalFree = modkernel32.NewProc("LocalFree") -) - -func ntCreateFile(handle *uintptr, accessMask uint32, oa *objectAttributes, iosb *ioStatusBlock, allocationSize *uint64, fileAttributes uint32, shareAccess uint32, createDisposition uint32, createOptions uint32, eaBuffer *byte, eaLength uint32) (status uint32) { - r0, _, _ := syscall.Syscall12(procNtCreateFile.Addr(), 11, uintptr(unsafe.Pointer(handle)), uintptr(accessMask), uintptr(unsafe.Pointer(oa)), uintptr(unsafe.Pointer(iosb)), uintptr(unsafe.Pointer(allocationSize)), uintptr(fileAttributes), uintptr(shareAccess), uintptr(createDisposition), uintptr(createOptions), uintptr(unsafe.Pointer(eaBuffer)), uintptr(eaLength), 0) - status = uint32(r0) - return -} - -func ntSetInformationFile(handle uintptr, iosb *ioStatusBlock, information uintptr, length uint32, class uint32) (status uint32) { - r0, _, _ := syscall.Syscall6(procNtSetInformationFile.Addr(), 5, uintptr(handle), uintptr(unsafe.Pointer(iosb)), uintptr(information), uintptr(length), uintptr(class), 0) - status = uint32(r0) - return -} - -func rtlNtStatusToDosError(status uint32) (winerr error) { - r0, _, _ := syscall.Syscall(procRtlNtStatusToDosErrorNoTeb.Addr(), 1, uintptr(status), 0, 0) - if r0 != 0 { - winerr = syscall.Errno(r0) - } - return -} - -func localAlloc(flags uint32, size int) (ptr uintptr) { - r0, _, _ := syscall.Syscall(procLocalAlloc.Addr(), 2, uintptr(flags), uintptr(size), 0) - ptr = uintptr(r0) - return -} - -func localFree(ptr uintptr) { - syscall.Syscall(procLocalFree.Addr(), 1, uintptr(ptr), 0, 0) - return -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema1/schema1.go b/vendor/github.com/Microsoft/hcsshim/internal/schema1/schema1.go deleted file mode 100644 index 995433ace..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema1/schema1.go +++ /dev/null @@ -1,245 +0,0 @@ -package schema1 - -import ( - "encoding/json" - "time" - - "github.com/Microsoft/hcsshim/internal/schema2" -) - -// ProcessConfig is used as both the input of Container.CreateProcess -// and to convert the parameters to JSON for passing onto the HCS -type ProcessConfig struct { - ApplicationName string `json:",omitempty"` - CommandLine string `json:",omitempty"` - CommandArgs []string `json:",omitempty"` // Used by Linux Containers on Windows - User string `json:",omitempty"` - WorkingDirectory string `json:",omitempty"` - Environment map[string]string `json:",omitempty"` - EmulateConsole bool `json:",omitempty"` - CreateStdInPipe bool `json:",omitempty"` - CreateStdOutPipe bool `json:",omitempty"` - CreateStdErrPipe bool `json:",omitempty"` - ConsoleSize [2]uint `json:",omitempty"` - CreateInUtilityVm bool `json:",omitempty"` // Used by Linux Containers on Windows - OCISpecification *json.RawMessage `json:",omitempty"` // Used by Linux Containers on Windows -} - -type Layer struct { - ID string - Path string -} - -type MappedDir struct { - HostPath string - ContainerPath string - ReadOnly bool - BandwidthMaximum uint64 - IOPSMaximum uint64 - CreateInUtilityVM bool - // LinuxMetadata - Support added in 1803/RS4+. - LinuxMetadata bool `json:",omitempty"` -} - -type MappedPipe struct { - HostPath string - ContainerPipeName string -} - -type HvRuntime struct { - ImagePath string `json:",omitempty"` - SkipTemplate bool `json:",omitempty"` - LinuxInitrdFile string `json:",omitempty"` // File under ImagePath on host containing an initrd image for starting a Linux utility VM - LinuxKernelFile string `json:",omitempty"` // File under ImagePath on host containing a kernel for starting a Linux utility VM - LinuxBootParameters string `json:",omitempty"` // Additional boot parameters for starting a Linux Utility VM in initrd mode - BootSource string `json:",omitempty"` // "Vhd" for Linux Utility VM booting from VHD - WritableBootSource bool `json:",omitempty"` // Linux Utility VM booting from VHD -} - -type MappedVirtualDisk struct { - HostPath string `json:",omitempty"` // Path to VHD on the host - ContainerPath string // Platform-specific mount point path in the container - CreateInUtilityVM bool `json:",omitempty"` - ReadOnly bool `json:",omitempty"` - Cache string `json:",omitempty"` // "" (Unspecified); "Disabled"; "Enabled"; "Private"; "PrivateAllowSharing" - AttachOnly bool `json:",omitempty:` -} - -// AssignedDevice represents a device that has been directly assigned to a container -// -// NOTE: Support added in RS5 -type AssignedDevice struct { - // InterfaceClassGUID of the device to assign to container. - InterfaceClassGUID string `json:"InterfaceClassGuid,omitempty"` -} - -// ContainerConfig is used as both the input of CreateContainer -// and to convert the parameters to JSON for passing onto the HCS -type ContainerConfig struct { - SystemType string // HCS requires this to be hard-coded to "Container" - Name string // Name of the container. We use the docker ID. - Owner string `json:",omitempty"` // The management platform that created this container - VolumePath string `json:",omitempty"` // Windows volume path for scratch space. Used by Windows Server Containers only. Format \\?\\Volume{GUID} - IgnoreFlushesDuringBoot bool `json:",omitempty"` // Optimization hint for container startup in Windows - LayerFolderPath string `json:",omitempty"` // Where the layer folders are located. Used by Windows Server Containers only. Format %root%\windowsfilter\containerID - Layers []Layer // List of storage layers. Required for Windows Server and Hyper-V Containers. Format ID=GUID;Path=%root%\windowsfilter\layerID - Credentials string `json:",omitempty"` // Credentials information - ProcessorCount uint32 `json:",omitempty"` // Number of processors to assign to the container. - ProcessorWeight uint64 `json:",omitempty"` // CPU shares (relative weight to other containers with cpu shares). Range is from 1 to 10000. A value of 0 results in default shares. - ProcessorMaximum int64 `json:",omitempty"` // Specifies the portion of processor cycles that this container can use as a percentage times 100. Range is from 1 to 10000. A value of 0 results in no limit. - StorageIOPSMaximum uint64 `json:",omitempty"` // Maximum Storage IOPS - StorageBandwidthMaximum uint64 `json:",omitempty"` // Maximum Storage Bandwidth in bytes per second - StorageSandboxSize uint64 `json:",omitempty"` // Size in bytes that the container system drive should be expanded to if smaller - MemoryMaximumInMB int64 `json:",omitempty"` // Maximum memory available to the container in Megabytes - HostName string `json:",omitempty"` // Hostname - MappedDirectories []MappedDir `json:",omitempty"` // List of mapped directories (volumes/mounts) - MappedPipes []MappedPipe `json:",omitempty"` // List of mapped Windows named pipes - HvPartition bool // True if it a Hyper-V Container - NetworkSharedContainerName string `json:",omitempty"` // Name (ID) of the container that we will share the network stack with. - EndpointList []string `json:",omitempty"` // List of networking endpoints to be attached to container - HvRuntime *HvRuntime `json:",omitempty"` // Hyper-V container settings. Used by Hyper-V containers only. Format ImagePath=%root%\BaseLayerID\UtilityVM - Servicing bool `json:",omitempty"` // True if this container is for servicing - AllowUnqualifiedDNSQuery bool `json:",omitempty"` // True to allow unqualified DNS name resolution - DNSSearchList string `json:",omitempty"` // Comma seperated list of DNS suffixes to use for name resolution - ContainerType string `json:",omitempty"` // "Linux" for Linux containers on Windows. Omitted otherwise. - TerminateOnLastHandleClosed bool `json:",omitempty"` // Should HCS terminate the container once all handles have been closed - MappedVirtualDisks []MappedVirtualDisk `json:",omitempty"` // Array of virtual disks to mount at start - AssignedDevices []AssignedDevice `json:",omitempty"` // Array of devices to assign. NOTE: Support added in RS5 -} - -type ComputeSystemQuery struct { - IDs []string `json:"Ids,omitempty"` - Types []string `json:",omitempty"` - Names []string `json:",omitempty"` - Owners []string `json:",omitempty"` -} - -type PropertyType string - -const ( - PropertyTypeStatistics PropertyType = "Statistics" // V1 and V2 - PropertyTypeProcessList = "ProcessList" // V1 and V2 - PropertyTypeMappedVirtualDisk = "MappedVirtualDisk" // Not supported in V2 schema call - PropertyTypeGuestConnection = "GuestConnection" // V1 and V2. Nil return from HCS before RS5 -) - -type PropertyQuery struct { - PropertyTypes []PropertyType `json:",omitempty"` -} - -// ContainerProperties holds the properties for a container and the processes running in that container -type ContainerProperties struct { - ID string `json:"Id"` - State string - Name string - SystemType string - Owner string - SiloGUID string `json:"SiloGuid,omitempty"` - RuntimeID string `json:"RuntimeId,omitempty"` - IsRuntimeTemplate bool `json:",omitempty"` - RuntimeImagePath string `json:",omitempty"` - Stopped bool `json:",omitempty"` - ExitType string `json:",omitempty"` - AreUpdatesPending bool `json:",omitempty"` - ObRoot string `json:",omitempty"` - Statistics Statistics `json:",omitempty"` - ProcessList []ProcessListItem `json:",omitempty"` - MappedVirtualDiskControllers map[int]MappedVirtualDiskController `json:",omitempty"` - GuestConnectionInfo GuestConnectionInfo `json:",omitempty"` -} - -// MemoryStats holds the memory statistics for a container -type MemoryStats struct { - UsageCommitBytes uint64 `json:"MemoryUsageCommitBytes,omitempty"` - UsageCommitPeakBytes uint64 `json:"MemoryUsageCommitPeakBytes,omitempty"` - UsagePrivateWorkingSetBytes uint64 `json:"MemoryUsagePrivateWorkingSetBytes,omitempty"` -} - -// ProcessorStats holds the processor statistics for a container -type ProcessorStats struct { - TotalRuntime100ns uint64 `json:",omitempty"` - RuntimeUser100ns uint64 `json:",omitempty"` - RuntimeKernel100ns uint64 `json:",omitempty"` -} - -// StorageStats holds the storage statistics for a container -type StorageStats struct { - ReadCountNormalized uint64 `json:",omitempty"` - ReadSizeBytes uint64 `json:",omitempty"` - WriteCountNormalized uint64 `json:",omitempty"` - WriteSizeBytes uint64 `json:",omitempty"` -} - -// NetworkStats holds the network statistics for a container -type NetworkStats struct { - BytesReceived uint64 `json:",omitempty"` - BytesSent uint64 `json:",omitempty"` - PacketsReceived uint64 `json:",omitempty"` - PacketsSent uint64 `json:",omitempty"` - DroppedPacketsIncoming uint64 `json:",omitempty"` - DroppedPacketsOutgoing uint64 `json:",omitempty"` - EndpointId string `json:",omitempty"` - InstanceId string `json:",omitempty"` -} - -// Statistics is the structure returned by a statistics call on a container -type Statistics struct { - Timestamp time.Time `json:",omitempty"` - ContainerStartTime time.Time `json:",omitempty"` - Uptime100ns uint64 `json:",omitempty"` - Memory MemoryStats `json:",omitempty"` - Processor ProcessorStats `json:",omitempty"` - Storage StorageStats `json:",omitempty"` - Network []NetworkStats `json:",omitempty"` -} - -// ProcessList is the structure of an item returned by a ProcessList call on a container -type ProcessListItem struct { - CreateTimestamp time.Time `json:",omitempty"` - ImageName string `json:",omitempty"` - KernelTime100ns uint64 `json:",omitempty"` - MemoryCommitBytes uint64 `json:",omitempty"` - MemoryWorkingSetPrivateBytes uint64 `json:",omitempty"` - MemoryWorkingSetSharedBytes uint64 `json:",omitempty"` - ProcessId uint32 `json:",omitempty"` - UserTime100ns uint64 `json:",omitempty"` -} - -// MappedVirtualDiskController is the structure of an item returned by a MappedVirtualDiskList call on a container -type MappedVirtualDiskController struct { - MappedVirtualDisks map[int]MappedVirtualDisk `json:",omitempty"` -} - -// GuestDefinedCapabilities is part of the GuestConnectionInfo returned by a GuestConnection call on a utility VM -type GuestDefinedCapabilities struct { - NamespaceAddRequestSupported bool `json:",omitempty"` - SignalProcessSupported bool `json:",omitempty"` -} - -// GuestConnectionInfo is the structure of an iterm return by a GuestConnection call on a utility VM -type GuestConnectionInfo struct { - SupportedSchemaVersions []hcsschema.Version `json:",omitempty"` - ProtocolVersion uint32 `json:",omitempty"` - GuestDefinedCapabilities GuestDefinedCapabilities `json:",omitempty"` -} - -// Type of Request Support in ModifySystem -type RequestType string - -// Type of Resource Support in ModifySystem -type ResourceType string - -// RequestType const -const ( - Add RequestType = "Add" - Remove RequestType = "Remove" - Network ResourceType = "Network" -) - -// ResourceModificationRequestResponse is the structure used to send request to the container to modify the system -// Supported resource types are Network and Request Types are Add/Remove -type ResourceModificationRequestResponse struct { - Resource ResourceType `json:"ResourceType"` - Data interface{} `json:"Settings"` - Request RequestType `json:"RequestType,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/attachment.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/attachment.go deleted file mode 100644 index 09456cbc2..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/attachment.go +++ /dev/null @@ -1,31 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type Attachment struct { - - Type_ string `json:"Type,omitempty"` - - Path string `json:"Path,omitempty"` - - IgnoreFlushes bool `json:"IgnoreFlushes,omitempty"` - - CachingMode string `json:"CachingMode,omitempty"` - - NoWriteHardening bool `json:"NoWriteHardening,omitempty"` - - DisableExpansionOptimization bool `json:"DisableExpansionOptimization,omitempty"` - - IgnoreRelativeLocator bool `json:"IgnoreRelativeLocator,omitempty"` - - CaptureIoAttributionContext bool `json:"CaptureIoAttributionContext,omitempty"` - - ReadOnly bool `json:"ReadOnly,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/battery.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/battery.go deleted file mode 100644 index ecbbed4c2..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/battery.go +++ /dev/null @@ -1,13 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type Battery struct { -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/cache_query_stats_response.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/cache_query_stats_response.go deleted file mode 100644 index 243779eab..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/cache_query_stats_response.go +++ /dev/null @@ -1,19 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type CacheQueryStatsResponse struct { - - L3OccupancyBytes int32 `json:"L3OccupancyBytes,omitempty"` - - L3TotalBwBytes int32 `json:"L3TotalBwBytes,omitempty"` - - L3LocalBwBytes int32 `json:"L3LocalBwBytes,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/chipset.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/chipset.go deleted file mode 100644 index ca75277a3..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/chipset.go +++ /dev/null @@ -1,27 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type Chipset struct { - Uefi *Uefi `json:"Uefi,omitempty"` - - IsNumLockDisabled bool `json:"IsNumLockDisabled,omitempty"` - - BaseBoardSerialNumber string `json:"BaseBoardSerialNumber,omitempty"` - - ChassisSerialNumber string `json:"ChassisSerialNumber,omitempty"` - - ChassisAssetTag string `json:"ChassisAssetTag,omitempty"` - - UseUtc bool `json:"UseUtc,omitempty"` - - // LinuxKernelDirect - Added in v2.2 Builds >=181117 - LinuxKernelDirect *LinuxKernelDirect `json:"LinuxKernelDirect,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/close_handle.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/close_handle.go deleted file mode 100644 index 88f01707a..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/close_handle.go +++ /dev/null @@ -1,15 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type CloseHandle struct { - - Handle string `json:"Handle,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/com_port.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/com_port.go deleted file mode 100644 index c665be3d5..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/com_port.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -// ComPort specifies the named pipe that will be used for the port, with empty string indicating a disconnected port. -type ComPort struct { - - NamedPipe string `json:"NamedPipe,omitempty"` - - OptimizeForDebugger bool `json:"OptimizeForDebugger,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/compute_system.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/compute_system.go deleted file mode 100644 index 85785d285..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/compute_system.go +++ /dev/null @@ -1,27 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type ComputeSystem struct { - - Owner string `json:"Owner,omitempty"` - - SchemaVersion *Version `json:"SchemaVersion,omitempty"` - - HostingSystemId string `json:"HostingSystemId,omitempty"` - - HostedSystem *HostedSystem `json:"HostedSystem,omitempty"` - - Container *Container `json:"Container,omitempty"` - - VirtualMachine *VirtualMachine `json:"VirtualMachine,omitempty"` - - ShouldTerminateOnLastHandleClosed bool `json:"ShouldTerminateOnLastHandleClosed,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/configuration.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/configuration.go deleted file mode 100644 index 1a47db7d9..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/configuration.go +++ /dev/null @@ -1,72 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -import ( - "net/http" -) - -// contextKeys are used to identify the type of value in the context. -// Since these are string, it is possible to get a short description of the -// context key for logging and debugging using key.String(). - -type contextKey string - -func (c contextKey) String() string { - return "auth " + string(c) -} - -var ( - // ContextOAuth2 takes a oauth2.TokenSource as authentication for the request. - ContextOAuth2 = contextKey("token") - - // ContextBasicAuth takes BasicAuth as authentication for the request. - ContextBasicAuth = contextKey("basic") - - // ContextAccessToken takes a string oauth2 access token as authentication for the request. - ContextAccessToken = contextKey("accesstoken") - - // ContextAPIKey takes an APIKey as authentication for the request - ContextAPIKey = contextKey("apikey") -) - -// BasicAuth provides basic http authentication to a request passed via context using ContextBasicAuth -type BasicAuth struct { - UserName string `json:"userName,omitempty"` - Password string `json:"password,omitempty"` -} - -// APIKey provides API key based authentication to a request passed via context using ContextAPIKey -type APIKey struct { - Key string - Prefix string -} - -type Configuration struct { - BasePath string `json:"basePath,omitempty"` - Host string `json:"host,omitempty"` - Scheme string `json:"scheme,omitempty"` - DefaultHeader map[string]string `json:"defaultHeader,omitempty"` - UserAgent string `json:"userAgent,omitempty"` - HTTPClient *http.Client -} - -func NewConfiguration() *Configuration { - cfg := &Configuration{ - BasePath: "https://localhost", - DefaultHeader: make(map[string]string), - UserAgent: "Swagger-Codegen/2.1.0/go", - } - return cfg -} - -func (c *Configuration) AddDefaultHeader(key string, value string) { - c.DefaultHeader[key] = value -} \ No newline at end of file diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/console_size.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/console_size.go deleted file mode 100644 index adbe07fe5..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/console_size.go +++ /dev/null @@ -1,17 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type ConsoleSize struct { - - Height int32 `json:"Height,omitempty"` - - Width int32 `json:"Width,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/container.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/container.go deleted file mode 100644 index 17dce28bc..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/container.go +++ /dev/null @@ -1,35 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type Container struct { - - GuestOs *GuestOs `json:"GuestOs,omitempty"` - - Storage *Storage `json:"Storage,omitempty"` - - MappedDirectories []MappedDirectory `json:"MappedDirectories,omitempty"` - - MappedPipes []MappedPipe `json:"MappedPipes,omitempty"` - - Memory *Memory `json:"Memory,omitempty"` - - Processor *Processor `json:"Processor,omitempty"` - - Networking *Networking `json:"Networking,omitempty"` - - HvSocket *HvSocket `json:"HvSocket,omitempty"` - - ContainerCredentialGuard *ContainerCredentialGuardState `json:"ContainerCredentialGuard,omitempty"` - - RegistryChanges *RegistryChanges `json:"RegistryChanges,omitempty"` - - AssignedDevices []Device `json:"AssignedDevices,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/container_credential_guard_state.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/container_credential_guard_state.go deleted file mode 100644 index 0f8f64437..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/container_credential_guard_state.go +++ /dev/null @@ -1,25 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type ContainerCredentialGuardState struct { - - // Authentication cookie for calls to a Container Credential Guard instance. - Cookie string `json:"Cookie,omitempty"` - - // Name of the RPC endpoint of the Container Credential Guard instance. - RpcEndpoint string `json:"RpcEndpoint,omitempty"` - - // Transport used for the configured Container Credential Guard instance. - Transport string `json:"Transport,omitempty"` - - // Credential spec used for the configured Container Credential Guard instance. - CredentialSpec string `json:"CredentialSpec,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/container_memory_information.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/container_memory_information.go deleted file mode 100644 index 754797e21..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/container_memory_information.go +++ /dev/null @@ -1,26 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -// memory usage as viewed from within the container -type ContainerMemoryInformation struct { - - TotalPhysicalBytes int32 `json:"TotalPhysicalBytes,omitempty"` - - TotalUsage int32 `json:"TotalUsage,omitempty"` - - CommittedBytes int32 `json:"CommittedBytes,omitempty"` - - SharedCommittedBytes int32 `json:"SharedCommittedBytes,omitempty"` - - CommitLimitBytes int32 `json:"CommitLimitBytes,omitempty"` - - PeakCommitmentBytes int32 `json:"PeakCommitmentBytes,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/device.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/device.go deleted file mode 100644 index ca319bbbc..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/device.go +++ /dev/null @@ -1,16 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type Device struct { - - // The interface class guid of the device to assign to container. - InterfaceClassGuid string `json:"InterfaceClassGuid,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/devices.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/devices.go deleted file mode 100644 index b2191c571..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/devices.go +++ /dev/null @@ -1,43 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type Devices struct { - - ComPorts map[string]ComPort `json:"ComPorts,omitempty"` - - Scsi map[string]Scsi `json:"Scsi,omitempty"` - - VirtualPMem *VirtualPMemController `json:"VirtualPMem,omitempty"` - - NetworkAdapters map[string]NetworkAdapter `json:"NetworkAdapters,omitempty"` - - VideoMonitor *VideoMonitor `json:"VideoMonitor,omitempty"` - - Keyboard *Keyboard `json:"Keyboard,omitempty"` - - Mouse *Mouse `json:"Mouse,omitempty"` - - HvSocket *HvSocket2 `json:"HvSocket,omitempty"` - - EnhancedModeVideo *EnhancedModeVideo `json:"EnhancedModeVideo,omitempty"` - - GuestCrashReporting *GuestCrashReporting `json:"GuestCrashReporting,omitempty"` - - VirtualSmb *VirtualSmb `json:"VirtualSmb,omitempty"` - - Plan9 *Plan9 `json:"Plan9,omitempty"` - - Battery *Battery `json:"Battery,omitempty"` - - FlexibleIov map[string]FlexibleIoDevice `json:"FlexibleIov,omitempty"` - - SharedMemory *SharedMemoryConfiguration `json:"SharedMemory,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/enhanced_mode_video.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/enhanced_mode_video.go deleted file mode 100644 index 4fe592f71..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/enhanced_mode_video.go +++ /dev/null @@ -1,15 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type EnhancedModeVideo struct { - - ConnectionOptions *RdpConnectionOptions `json:"ConnectionOptions,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/flexible_io_device.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/flexible_io_device.go deleted file mode 100644 index 51011afe4..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/flexible_io_device.go +++ /dev/null @@ -1,19 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type FlexibleIoDevice struct { - - EmulatorId string `json:"EmulatorId,omitempty"` - - HostingModel string `json:"HostingModel,omitempty"` - - Configuration []string `json:"Configuration,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_connection.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_connection.go deleted file mode 100644 index 7db29495b..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_connection.go +++ /dev/null @@ -1,19 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type GuestConnection struct { - - // Use Vsock rather than Hyper-V sockets to communicate with the guest service. - UseVsock bool `json:"UseVsock,omitempty"` - - // Don't disconnect the guest connection when pausing the virtual machine. - UseConnectedSuspend bool `json:"UseConnectedSuspend,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_connection_info.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_connection_info.go deleted file mode 100644 index 8a369bab7..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_connection_info.go +++ /dev/null @@ -1,21 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -// Information about the guest. -type GuestConnectionInfo struct { - - // Each schema version x.y stands for the range of versions a.b where a==x and b<=y. This list comes from the SupportedSchemaVersions field in GcsCapabilities. - SupportedSchemaVersions []Version `json:"SupportedSchemaVersions,omitempty"` - - ProtocolVersion int32 `json:"ProtocolVersion,omitempty"` - - GuestDefinedCapabilities *interface{} `json:"GuestDefinedCapabilities,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_crash_reporting.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_crash_reporting.go deleted file mode 100644 index c5fa76735..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_crash_reporting.go +++ /dev/null @@ -1,15 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type GuestCrashReporting struct { - - WindowsCrashSettings *WindowsCrashReporting `json:"WindowsCrashSettings,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_os.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_os.go deleted file mode 100644 index c708fc7c3..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_os.go +++ /dev/null @@ -1,15 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type GuestOs struct { - - HostName string `json:"HostName,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_state.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_state.go deleted file mode 100644 index ef1eec886..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/guest_state.go +++ /dev/null @@ -1,22 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type GuestState struct { - - // The path to an existing file uses for persistent guest state storage. An empty string indicates the system should initialize new transient, in-memory guest state. - GuestStateFilePath string `json:"GuestStateFilePath,omitempty"` - - // The path to an existing file for persistent runtime state storage. An empty string indicates the system should initialize new transient, in-memory runtime state. - RuntimeStateFilePath string `json:"RuntimeStateFilePath,omitempty"` - - // If true, the guest state and runtime state files will be used as templates to populate transient, in-memory state instead of using the files as persistent backing store. - ForceTransientState bool `json:"ForceTransientState,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/hosted_system.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/hosted_system.go deleted file mode 100644 index 0797584c5..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/hosted_system.go +++ /dev/null @@ -1,17 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type HostedSystem struct { - - SchemaVersion *Version `json:"SchemaVersion,omitempty"` - - Container *Container `json:"Container,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket.go deleted file mode 100644 index ef9ffb8dd..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket.go +++ /dev/null @@ -1,17 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type HvSocket struct { - - Config *HvSocketSystemConfig `json:"Config,omitempty"` - - EnablePowerShellDirect bool `json:"EnablePowerShellDirect,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket_2.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket_2.go deleted file mode 100644 index a19ba15c1..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket_2.go +++ /dev/null @@ -1,16 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -// HvSocket configuration for a VM -type HvSocket2 struct { - - HvSocketConfig *HvSocketSystemConfig `json:"HvSocketConfig,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket_service_config.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket_service_config.go deleted file mode 100644 index a848e91e6..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket_service_config.go +++ /dev/null @@ -1,22 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type HvSocketServiceConfig struct { - - // SDDL string that HvSocket will check before allowing a host process to bind to this specific service. If not specified, defaults to the system DefaultBindSecurityDescriptor, defined in HvSocketSystemWpConfig in V1. - BindSecurityDescriptor string `json:"BindSecurityDescriptor,omitempty"` - - // SDDL string that HvSocket will check before allowing a host process to connect to this specific service. If not specified, defaults to the system DefaultConnectSecurityDescriptor, defined in HvSocketSystemWpConfig in V1. - ConnectSecurityDescriptor string `json:"ConnectSecurityDescriptor,omitempty"` - - // If true, HvSocket will process wildcard binds for this service/system combination. Wildcard binds are secured in the registry at SOFTWARE/Microsoft/Windows NT/CurrentVersion/Virtualization/HvSocket/WildcardDescriptors - AllowWildcardBinds bool `json:"AllowWildcardBinds,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket_system_config.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket_system_config.go deleted file mode 100644 index 69f4f9d39..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/hv_socket_system_config.go +++ /dev/null @@ -1,22 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -// This is the HCS Schema version of the HvSocket configuration. The VMWP version is located in Config.Devices.IC in V1. -type HvSocketSystemConfig struct { - - // SDDL string that HvSocket will check before allowing a host process to bind to an unlisted service for this specific container/VM (not wildcard binds). - DefaultBindSecurityDescriptor string `json:"DefaultBindSecurityDescriptor,omitempty"` - - // SDDL string that HvSocket will check before allowing a host process to connect to an unlisted service in the VM/container. - DefaultConnectSecurityDescriptor string `json:"DefaultConnectSecurityDescriptor,omitempty"` - - ServiceTable map[string]HvSocketServiceConfig `json:"ServiceTable,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/keyboard.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/keyboard.go deleted file mode 100644 index 3d3fa3b1c..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/keyboard.go +++ /dev/null @@ -1,13 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type Keyboard struct { -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/layer.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/layer.go deleted file mode 100644 index b63b8ef12..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/layer.go +++ /dev/null @@ -1,22 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type Layer struct { - - Id string `json:"Id,omitempty"` - - Path string `json:"Path,omitempty"` - - PathType string `json:"PathType,omitempty"` - - // Unspecified defaults to Enabled - Cache string `json:"Cache,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/linux_kernel_direct.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/linux_kernel_direct.go deleted file mode 100644 index 0ab6c280f..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/linux_kernel_direct.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.2 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type LinuxKernelDirect struct { - KernelFilePath string `json:"KernelFilePath,omitempty"` - - InitRdPath string `json:"InitRdPath,omitempty"` - - KernelCmdLine string `json:"KernelCmdLine,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/mapped_directory.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/mapped_directory.go deleted file mode 100644 index a823a6d3b..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/mapped_directory.go +++ /dev/null @@ -1,21 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type MappedDirectory struct { - - HostPath string `json:"HostPath,omitempty"` - - HostPathType string `json:"HostPathType,omitempty"` - - ContainerPath string `json:"ContainerPath,omitempty"` - - ReadOnly bool `json:"ReadOnly,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/mapped_pipe.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/mapped_pipe.go deleted file mode 100644 index 2d1d2604a..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/mapped_pipe.go +++ /dev/null @@ -1,19 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type MappedPipe struct { - - ContainerPipeName string `json:"ContainerPipeName,omitempty"` - - HostPath string `json:"HostPath,omitempty"` - - HostPathType string `json:"HostPathType,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/memory.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/memory.go deleted file mode 100644 index e1d135a3a..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/memory.go +++ /dev/null @@ -1,15 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type Memory struct { - - SizeInMB int32 `json:"SizeInMB,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/memory_2.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/memory_2.go deleted file mode 100644 index 27d0b8c48..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/memory_2.go +++ /dev/null @@ -1,25 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type Memory2 struct { - SizeInMB int32 `json:"SizeInMB,omitempty"` - - AllowOvercommit bool `json:"AllowOvercommit,omitempty"` - - EnableHotHint bool `json:"EnableHotHint,omitempty"` - - EnableColdHint bool `json:"EnableColdHint,omitempty"` - - EnableEpf bool `json:"EnableEpf,omitempty"` - - // EnableDeferredCommit is private in the schema. If regenerated need to add back. - EnableDeferredCommit bool `json:"EnableDeferredCommit,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/memory_information_for_vm.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/memory_information_for_vm.go deleted file mode 100644 index bdd87dffd..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/memory_information_for_vm.go +++ /dev/null @@ -1,19 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type MemoryInformationForVm struct { - - VirtualNodeCount int32 `json:"VirtualNodeCount,omitempty"` - - VirtualMachineMemory *VmMemory `json:"VirtualMachineMemory,omitempty"` - - VirtualNodes []VirtualNodeInfo `json:"VirtualNodes,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/memory_stats.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/memory_stats.go deleted file mode 100644 index 6214970f6..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/memory_stats.go +++ /dev/null @@ -1,20 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -// Memory runtime statistics -type MemoryStats struct { - - MemoryUsageCommitBytes int32 `json:"MemoryUsageCommitBytes,omitempty"` - - MemoryUsageCommitPeakBytes int32 `json:"MemoryUsageCommitPeakBytes,omitempty"` - - MemoryUsagePrivateWorkingSetBytes int32 `json:"MemoryUsagePrivateWorkingSetBytes,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/modify_setting_request.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/modify_setting_request.go deleted file mode 100644 index d29455a3e..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/modify_setting_request.go +++ /dev/null @@ -1,20 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type ModifySettingRequest struct { - ResourcePath string `json:"ResourcePath,omitempty"` - - RequestType string `json:"RequestType,omitempty"` - - Settings interface{} `json:"Settings,omitempty"` // NOTE: Swagger generated as *interface{}. Locally updated - - GuestRequest interface{} `json:"GuestRequest,omitempty"` // NOTE: Swagger generated as *interface{}. Locally updated -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/mouse.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/mouse.go deleted file mode 100644 index ccf8b938f..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/mouse.go +++ /dev/null @@ -1,13 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type Mouse struct { -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/network_adapter.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/network_adapter.go deleted file mode 100644 index c586f66c2..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/network_adapter.go +++ /dev/null @@ -1,17 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type NetworkAdapter struct { - - EndpointId string `json:"EndpointId,omitempty"` - - MacAddress string `json:"MacAddress,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/networking.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/networking.go deleted file mode 100644 index 12c47827c..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/networking.go +++ /dev/null @@ -1,24 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type Networking struct { - - AllowUnqualifiedDnsQuery bool `json:"AllowUnqualifiedDnsQuery,omitempty"` - - DnsSearchList string `json:"DnsSearchList,omitempty"` - - NetworkSharedContainerName string `json:"NetworkSharedContainerName,omitempty"` - - // Guid in windows; string in linux - Namespace string `json:"Namespace,omitempty"` - - NetworkAdapters []string `json:"NetworkAdapters,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/pause_notification.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/pause_notification.go deleted file mode 100644 index 1cd70d179..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/pause_notification.go +++ /dev/null @@ -1,16 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -// Notification data that is indicated to components running in the Virtual Machine. -type PauseNotification struct { - - Reason string `json:"Reason,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/pause_options.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/pause_options.go deleted file mode 100644 index 780a5cae2..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/pause_options.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -// Options for HcsPauseComputeSystem -type PauseOptions struct { - - SuspensionLevel string `json:"SuspensionLevel,omitempty"` - - HostedNotification *PauseNotification `json:"HostedNotification,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/plan9.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/plan9.go deleted file mode 100644 index 705c677e1..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/plan9.go +++ /dev/null @@ -1,15 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type Plan9 struct { - - Shares []Plan9Share `json:"Shares,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/plan9_share.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/plan9_share.go deleted file mode 100644 index eb171817a..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/plan9_share.go +++ /dev/null @@ -1,33 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type Plan9Share struct { - - Name string `json:"Name,omitempty"` - - // The name by which the guest operation system can access this share, via the aname parameter in the Plan9 protocol. - AccessName string `json:"AccessName,omitempty"` - - Path string `json:"Path,omitempty"` - - Port int32 `json:"Port,omitempty"` - - // Flags are marked private. Until they are exported correctly - // - // ReadOnly 0x00000001 - // LinuxMetadata 0x00000004 - // CaseSensitive 0x00000008 - Flags int32 `json:"Flags,omitempty"` - - ReadOnly bool `json:"ReadOnly,omitempty"` - - UseShareRootIdentity bool `json:"UseShareRootIdentity,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/process_details.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/process_details.go deleted file mode 100644 index 63e0b7f8f..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/process_details.go +++ /dev/null @@ -1,34 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -import ( - "time" -) - -// Information about a process running in a container -type ProcessDetails struct { - - ProcessId int32 `json:"ProcessId,omitempty"` - - ImageName string `json:"ImageName,omitempty"` - - CreateTimestamp time.Time `json:"CreateTimestamp,omitempty"` - - UserTime100ns int32 `json:"UserTime100ns,omitempty"` - - KernelTime100ns int32 `json:"KernelTime100ns,omitempty"` - - MemoryCommitBytes int32 `json:"MemoryCommitBytes,omitempty"` - - MemoryWorkingSetPrivateBytes int32 `json:"MemoryWorkingSetPrivateBytes,omitempty"` - - MemoryWorkingSetSharedBytes int32 `json:"MemoryWorkingSetSharedBytes,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/process_modify_request.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/process_modify_request.go deleted file mode 100644 index 29bc2e3d0..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/process_modify_request.go +++ /dev/null @@ -1,20 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -// Passed to HcsRpc_ModifyProcess -type ProcessModifyRequest struct { - - Operation string `json:"Operation,omitempty"` - - ConsoleSize *ConsoleSize `json:"ConsoleSize,omitempty"` - - CloseHandle *CloseHandle `json:"CloseHandle,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/process_parameters.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/process_parameters.go deleted file mode 100644 index 470c55734..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/process_parameters.go +++ /dev/null @@ -1,47 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type ProcessParameters struct { - - ApplicationName string `json:"ApplicationName,omitempty"` - - CommandLine string `json:"CommandLine,omitempty"` - - // optional alternative to CommandLine, currently only supported by Linux GCS - CommandArgs []string `json:"CommandArgs,omitempty"` - - User string `json:"User,omitempty"` - - WorkingDirectory string `json:"WorkingDirectory,omitempty"` - - Environment map[string]string `json:"Environment,omitempty"` - - // if set, will run as low-privilege process - RestrictedToken bool `json:"RestrictedToken,omitempty"` - - // if set, ignore StdErrPipe - EmulateConsole bool `json:"EmulateConsole,omitempty"` - - CreateStdInPipe bool `json:"CreateStdInPipe,omitempty"` - - CreateStdOutPipe bool `json:"CreateStdOutPipe,omitempty"` - - CreateStdErrPipe bool `json:"CreateStdErrPipe,omitempty"` - - // height then width - ConsoleSize []int32 `json:"ConsoleSize,omitempty"` - - // if set, find an existing session for the user and create the process in it - UseExistingLogin bool `json:"UseExistingLogin,omitempty"` - - // if set, use the legacy console instead of conhost - UseLegacyConsole bool `json:"UseLegacyConsole,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/process_status.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/process_status.go deleted file mode 100644 index 20793d150..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/process_status.go +++ /dev/null @@ -1,22 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -// Status of a process running in a container -type ProcessStatus struct { - - ProcessId int32 `json:"ProcessId,omitempty"` - - Exited bool `json:"Exited,omitempty"` - - ExitCode int32 `json:"ExitCode,omitempty"` - - LastWaitResult int32 `json:"LastWaitResult,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/processor.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/processor.go deleted file mode 100644 index 7a60b0245..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/processor.go +++ /dev/null @@ -1,19 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type Processor struct { - - Count int32 `json:"Count,omitempty"` - - Maximum int32 `json:"Maximum,omitempty"` - - Weight int32 `json:"Weight,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/processor_2.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/processor_2.go deleted file mode 100644 index 40d3e7356..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/processor_2.go +++ /dev/null @@ -1,21 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type Processor2 struct { - - Count int32 `json:"Count,omitempty"` - - Limit int32 `json:"Limit,omitempty"` - - Weight int32 `json:"Weight,omitempty"` - - ExposeVirtualizationExtensions bool `json:"ExposeVirtualizationExtensions,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/processor_stats.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/processor_stats.go deleted file mode 100644 index 9d3b77e57..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/processor_stats.go +++ /dev/null @@ -1,20 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -// CPU runtime statistics -type ProcessorStats struct { - - TotalRuntime100ns int32 `json:"TotalRuntime100ns,omitempty"` - - RuntimeUser100ns int32 `json:"RuntimeUser100ns,omitempty"` - - RuntimeKernel100ns int32 `json:"RuntimeKernel100ns,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/properties.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/properties.go deleted file mode 100644 index 6db2a48f6..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/properties.go +++ /dev/null @@ -1,47 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type Properties struct { - - Id string `json:"Id,omitempty"` - - SystemType string `json:"SystemType,omitempty"` - - RuntimeOsType string `json:"RuntimeOsType,omitempty"` - - Name string `json:"Name,omitempty"` - - Owner string `json:"Owner,omitempty"` - - RuntimeId string `json:"RuntimeId,omitempty"` - - RuntimeTemplateId string `json:"RuntimeTemplateId,omitempty"` - - State string `json:"State,omitempty"` - - Stopped bool `json:"Stopped,omitempty"` - - ExitType string `json:"ExitType,omitempty"` - - Memory *MemoryInformationForVm `json:"Memory,omitempty"` - - Statistics *Statistics `json:"Statistics,omitempty"` - - ProcessList []ProcessDetails `json:"ProcessList,omitempty"` - - TerminateOnLastHandleClosed bool `json:"TerminateOnLastHandleClosed,omitempty"` - - HostingSystemId string `json:"HostingSystemId,omitempty"` - - SharedMemoryRegionInfo []SharedMemoryRegionInfo `json:"SharedMemoryRegionInfo,omitempty"` - - GuestConnectionInfo *GuestConnectionInfo `json:"GuestConnectionInfo,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/property_query.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/property_query.go deleted file mode 100644 index 22b92ffdf..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/property_query.go +++ /dev/null @@ -1,16 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -// By default the basic properties will be returned. This query provides a way to request specific properties. -type PropertyQuery struct { - - PropertyTypes []string `json:"PropertyTypes,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/rdp_connection_options.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/rdp_connection_options.go deleted file mode 100644 index 97e453128..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/rdp_connection_options.go +++ /dev/null @@ -1,17 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type RdpConnectionOptions struct { - - AccessSids []string `json:"AccessSids,omitempty"` - - NamedPipe string `json:"NamedPipe,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/registry_changes.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/registry_changes.go deleted file mode 100644 index fa574ccc8..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/registry_changes.go +++ /dev/null @@ -1,17 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type RegistryChanges struct { - - AddValues []RegistryValue `json:"AddValues,omitempty"` - - DeleteKeys []RegistryKey `json:"DeleteKeys,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/registry_key.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/registry_key.go deleted file mode 100644 index fab03bc60..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/registry_key.go +++ /dev/null @@ -1,19 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type RegistryKey struct { - - Hive string `json:"Hive,omitempty"` - - Name string `json:"Name,omitempty"` - - Volatile bool `json:"Volatile,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/registry_value.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/registry_value.go deleted file mode 100644 index 1589f4841..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/registry_value.go +++ /dev/null @@ -1,31 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type RegistryValue struct { - - Key *RegistryKey `json:"Key,omitempty"` - - Name string `json:"Name,omitempty"` - - Type_ string `json:"Type,omitempty"` - - // One and only one value type must be set. - StringValue string `json:"StringValue,omitempty"` - - BinaryValue string `json:"BinaryValue,omitempty"` - - DWordValue int32 `json:"DWordValue,omitempty"` - - QWordValue int32 `json:"QWordValue,omitempty"` - - // Only used if RegistryValueType is CustomType The data is in BinaryValue - CustomType int32 `json:"CustomType,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/restore_state.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/restore_state.go deleted file mode 100644 index 778ff5873..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/restore_state.go +++ /dev/null @@ -1,19 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type RestoreState struct { - - // The path to the save state file to restore the system from. - SaveStateFilePath string `json:"SaveStateFilePath,omitempty"` - - // The ID of the template system to clone this new system off of. An empty string indicates the system should not be cloned from a template. - TemplateSystemId string `json:"TemplateSystemId,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/save_options.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/save_options.go deleted file mode 100644 index e55fa1d98..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/save_options.go +++ /dev/null @@ -1,19 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type SaveOptions struct { - - // The type of save operation to be performed. - SaveType string `json:"SaveType,omitempty"` - - // The path to the file that will container the saved state. - SaveStateFilePath string `json:"SaveStateFilePath,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/scsi.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/scsi.go deleted file mode 100644 index bf253a470..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/scsi.go +++ /dev/null @@ -1,16 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type Scsi struct { - - // Map of attachments, where the key is the integer LUN number on the controller. - Attachments map[string]Attachment `json:"Attachments,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/shared_memory_configuration.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/shared_memory_configuration.go deleted file mode 100644 index bd573f6cd..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/shared_memory_configuration.go +++ /dev/null @@ -1,15 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type SharedMemoryConfiguration struct { - - Regions []SharedMemoryRegion `json:"Regions,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/shared_memory_region.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/shared_memory_region.go deleted file mode 100644 index a57b2cba7..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/shared_memory_region.go +++ /dev/null @@ -1,23 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type SharedMemoryRegion struct { - - SectionName string `json:"SectionName,omitempty"` - - StartOffset int32 `json:"StartOffset,omitempty"` - - Length int32 `json:"Length,omitempty"` - - AllowGuestWrite bool `json:"AllowGuestWrite,omitempty"` - - HiddenFromGuest bool `json:"HiddenFromGuest,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/shared_memory_region_info.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/shared_memory_region_info.go deleted file mode 100644 index d9a50cc7d..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/shared_memory_region_info.go +++ /dev/null @@ -1,17 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type SharedMemoryRegionInfo struct { - - SectionName string `json:"SectionName,omitempty"` - - GuestPhysicalAddress int32 `json:"GuestPhysicalAddress,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/silo_properties.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/silo_properties.go deleted file mode 100644 index 599c06e8a..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/silo_properties.go +++ /dev/null @@ -1,18 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -// Silo job information -type SiloProperties struct { - - Enabled bool `json:"Enabled,omitempty"` - - JobName string `json:"JobName,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/statistics.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/statistics.go deleted file mode 100644 index 5cb3ed93b..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/statistics.go +++ /dev/null @@ -1,30 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -import ( - "time" -) - -// Runtime statistics for a container -type Statistics struct { - - Timestamp time.Time `json:"Timestamp,omitempty"` - - ContainerStartTime time.Time `json:"ContainerStartTime,omitempty"` - - Uptime100ns int32 `json:"Uptime100ns,omitempty"` - - Processor *ProcessorStats `json:"Processor,omitempty"` - - Memory *MemoryStats `json:"Memory,omitempty"` - - Storage *StorageStats `json:"Storage,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/storage.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/storage.go deleted file mode 100644 index 2627af913..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/storage.go +++ /dev/null @@ -1,21 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type Storage struct { - - // List of layers that describe the parent hierarchy for a container's storage. These layers combined together, presented as a disposable and/or committable working storage, are used by the container to record all changes done to the parent layers. - Layers []Layer `json:"Layers,omitempty"` - - // Path that points to the scratch space of a container, where parent layers are combined together to present a new disposable and/or committable layer with the changes done during its runtime. - Path string `json:"Path,omitempty"` - - QoS *StorageQoS `json:"QoS,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/storage_qo_s.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/storage_qo_s.go deleted file mode 100644 index 8c5255df1..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/storage_qo_s.go +++ /dev/null @@ -1,17 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type StorageQoS struct { - - IopsMaximum int32 `json:"IopsMaximum,omitempty"` - - BandwidthMaximum int32 `json:"BandwidthMaximum,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/storage_stats.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/storage_stats.go deleted file mode 100644 index 198ea57d7..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/storage_stats.go +++ /dev/null @@ -1,22 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -// Storage runtime statistics -type StorageStats struct { - - ReadCountNormalized int32 `json:"ReadCountNormalized,omitempty"` - - ReadSizeBytes int32 `json:"ReadSizeBytes,omitempty"` - - WriteCountNormalized int32 `json:"WriteCountNormalized,omitempty"` - - WriteSizeBytes int32 `json:"WriteSizeBytes,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/topology.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/topology.go deleted file mode 100644 index af2e3c823..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/topology.go +++ /dev/null @@ -1,17 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type Topology struct { - - Memory *Memory2 `json:"Memory,omitempty"` - - Processor *Processor2 `json:"Processor,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/uefi.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/uefi.go deleted file mode 100644 index ba91178f9..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/uefi.go +++ /dev/null @@ -1,21 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type Uefi struct { - - EnableDebugger bool `json:"EnableDebugger,omitempty"` - - SecureBootTemplateId string `json:"SecureBootTemplateId,omitempty"` - - BootThis *UefiBootEntry `json:"BootThis,omitempty"` - - Console string `json:"Console,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/uefi_boot_entry.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/uefi_boot_entry.go deleted file mode 100644 index 6620fb2bc..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/uefi_boot_entry.go +++ /dev/null @@ -1,23 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type UefiBootEntry struct { - - DeviceType string `json:"DeviceType,omitempty"` - - DevicePath string `json:"DevicePath,omitempty"` - - DiskNumber int32 `json:"DiskNumber,omitempty"` - - OptionalData string `json:"OptionalData,omitempty"` - - VmbFsRootPath string `json:"VmbFsRootPath,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/version.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/version.go deleted file mode 100644 index 62c0e4d12..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/version.go +++ /dev/null @@ -1,17 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type Version struct { - - Major int32 `json:"Major,omitempty"` - - Minor int32 `json:"Minor,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/video_monitor.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/video_monitor.go deleted file mode 100644 index 0958e5606..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/video_monitor.go +++ /dev/null @@ -1,19 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type VideoMonitor struct { - - HorizontalResolution int32 `json:"HorizontalResolution,omitempty"` - - VerticalResolution int32 `json:"VerticalResolution,omitempty"` - - ConnectionOptions *RdpConnectionOptions `json:"ConnectionOptions,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_machine.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_machine.go deleted file mode 100644 index 2d22b1bcb..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_machine.go +++ /dev/null @@ -1,32 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type VirtualMachine struct { - - // StopOnReset is private in the schema. If regenerated need to put back. - StopOnReset bool `json:"StopOnReset,omitempty"` - - Chipset *Chipset `json:"Chipset,omitempty"` - - ComputeTopology *Topology `json:"ComputeTopology,omitempty"` - - Devices *Devices `json:"Devices,omitempty"` - - GuestState *GuestState `json:"GuestState,omitempty"` - - RestoreState *RestoreState `json:"RestoreState,omitempty"` - - RegistryChanges *RegistryChanges `json:"RegistryChanges,omitempty"` - - StorageQoS *StorageQoS `json:"StorageQoS,omitempty"` - - GuestConnection *GuestConnection `json:"GuestConnection,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_node_info.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_node_info.go deleted file mode 100644 index 48402d8ec..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_node_info.go +++ /dev/null @@ -1,21 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type VirtualNodeInfo struct { - - VirtualNodeIndex int32 `json:"VirtualNodeIndex,omitempty"` - - PhysicalNodeNumber int32 `json:"PhysicalNodeNumber,omitempty"` - - VirtualProcessorCount int32 `json:"VirtualProcessorCount,omitempty"` - - MemoryUsageInPages int32 `json:"MemoryUsageInPages,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_p_mem_controller.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_p_mem_controller.go deleted file mode 100644 index f5b7f3e38..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_p_mem_controller.go +++ /dev/null @@ -1,20 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type VirtualPMemController struct { - Devices map[string]VirtualPMemDevice `json:"Devices,omitempty"` - - MaximumCount uint32 `json:"MaximumCount,omitempty"` - - MaximumSizeBytes uint64 `json:"MaximumSizeBytes,omitempty"` - - Backing string `json:"Backing,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_p_mem_device.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_p_mem_device.go deleted file mode 100644 index 47714444a..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_p_mem_device.go +++ /dev/null @@ -1,19 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type VirtualPMemDevice struct { - - HostPath string `json:"HostPath,omitempty"` - - ReadOnly bool `json:"ReadOnly,omitempty"` - - ImageFormat string `json:"ImageFormat,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_smb.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_smb.go deleted file mode 100644 index 76131b3a7..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_smb.go +++ /dev/null @@ -1,17 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type VirtualSmb struct { - - Shares []VirtualSmbShare `json:"Shares,omitempty"` - - DirectFileMappingInMB int64 `json:"DirectFileMappingInMB,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_smb_share.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_smb_share.go deleted file mode 100644 index b50098a42..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_smb_share.go +++ /dev/null @@ -1,21 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type VirtualSmbShare struct { - - Name string `json:"Name,omitempty"` - - Path string `json:"Path,omitempty"` - - AllowedFiles []string `json:"AllowedFiles,omitempty"` - - Options *VirtualSmbShareOptions `json:"Options,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_smb_share_options.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_smb_share_options.go deleted file mode 100644 index c1894279d..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/virtual_smb_share_options.go +++ /dev/null @@ -1,63 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type VirtualSmbShareOptions struct { - - ReadOnly bool `json:"ReadOnly,omitempty"` - - // convert exclusive access to shared read access - ShareRead bool `json:"ShareRead,omitempty"` - - // all opens will use cached I/O - CacheIo bool `json:"CacheIo,omitempty"` - - // disable oplock support - NoOplocks bool `json:"NoOplocks,omitempty"` - - // Acquire the backup privilege when attempting to open - TakeBackupPrivilege bool `json:"TakeBackupPrivilege,omitempty"` - - // Use the identity of the share root when opening - UseShareRootIdentity bool `json:"UseShareRootIdentity,omitempty"` - - // disable Direct Mapping - NoDirectmap bool `json:"NoDirectmap,omitempty"` - - // disable Byterange locks - NoLocks bool `json:"NoLocks,omitempty"` - - // disable Directory CHange Notifications - NoDirnotify bool `json:"NoDirnotify,omitempty"` - - // share is use for VM shared memory - VmSharedMemory bool `json:"VmSharedMemory,omitempty"` - - // allow access only to the files specified in AllowedFiles - RestrictFileAccess bool `json:"RestrictFileAccess,omitempty"` - - // disable all oplocks except Level II - ForceLevelIIOplocks bool `json:"ForceLevelIIOplocks,omitempty"` - - // Allow the host to reparse this base layer - ReparseBaseLayer bool `json:"ReparseBaseLayer,omitempty"` - - // Enable pseudo-oplocks - PseudoOplocks bool `json:"PseudoOplocks,omitempty"` - - // All opens will use non-cached IO - NonCacheIo bool `json:"NonCacheIo,omitempty"` - - // Enable pseudo directory change notifications - PseudoDirnotify bool `json:"PseudoDirnotify,omitempty"` - - // Block directory enumeration, renames, and deletes. - SingleFileMapping bool `json:"SingleFileMapping,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/vm_memory.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/vm_memory.go deleted file mode 100644 index 39f628667..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/vm_memory.go +++ /dev/null @@ -1,27 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type VmMemory struct { - - AvailableMemory int32 `json:"AvailableMemory,omitempty"` - - AvailableMemoryBuffer int32 `json:"AvailableMemoryBuffer,omitempty"` - - ReservedMemory int32 `json:"ReservedMemory,omitempty"` - - AssignedMemory int32 `json:"AssignedMemory,omitempty"` - - SlpActive bool `json:"SlpActive,omitempty"` - - BalancingEnabled bool `json:"BalancingEnabled,omitempty"` - - DmOperationInProgress bool `json:"DmOperationInProgress,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/windows_crash_reporting.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/windows_crash_reporting.go deleted file mode 100644 index cf632bbc8..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/windows_crash_reporting.go +++ /dev/null @@ -1,17 +0,0 @@ -/* - * HCS API - * - * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) - * - * API version: 2.1 - * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) - */ - -package hcsschema - -type WindowsCrashReporting struct { - - DumpFileName string `json:"DumpFileName,omitempty"` - - MaxDumpSize int64 `json:"MaxDumpSize,omitempty"` -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/timeout/timeout.go b/vendor/github.com/Microsoft/hcsshim/internal/timeout/timeout.go deleted file mode 100644 index ff3b6572e..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/timeout/timeout.go +++ /dev/null @@ -1,70 +0,0 @@ -package timeout - -import ( - "os" - "strconv" - "time" -) - -var ( - // defaultTimeout is the timeout for most operations that is not overridden. - defaultTimeout = 4 * time.Minute - - // defaultTimeoutTestdRetry is the retry loop timeout for testd to respond - // for a disk to come online in LCOW. - defaultTimeoutTestdRetry = 5 * time.Second -) - -// External variables for HCSShim consumers to use. -var ( - // SystemCreate is the timeout for creating a compute system - SystemCreate time.Duration = defaultTimeout - - // SystemStart is the timeout for starting a compute system - SystemStart time.Duration = defaultTimeout - - // SystemPause is the timeout for pausing a compute system - SystemPause time.Duration = defaultTimeout - - // SystemResume is the timeout for resuming a compute system - SystemResume time.Duration = defaultTimeout - - // SyscallWatcher is the timeout before warning of a potential stuck platform syscall. - SyscallWatcher time.Duration = defaultTimeout - - // Tar2VHD is the timeout for the tar2vhd operation to complete - Tar2VHD time.Duration = defaultTimeout - - // ExternalCommandToStart is the timeout for external commands to start - ExternalCommandToStart = defaultTimeout - - // ExternalCommandToComplete is the timeout for external commands to complete. - // Generally this means copying data from their stdio pipes. - ExternalCommandToComplete = defaultTimeout - - // TestDRetryLoop is the timeout for testd retry loop when onlining a SCSI disk in LCOW - TestDRetryLoop = defaultTimeoutTestdRetry -) - -func init() { - SystemCreate = durationFromEnvironment("HCSSHIM_TIMEOUT_SYSTEMCREATE", SystemCreate) - SystemStart = durationFromEnvironment("HCSSHIM_TIMEOUT_SYSTEMSTART", SystemStart) - SystemPause = durationFromEnvironment("HCSSHIM_TIMEOUT_SYSTEMPAUSE", SystemPause) - SystemResume = durationFromEnvironment("HCSSHIM_TIMEOUT_SYSTEMRESUME", SystemResume) - SyscallWatcher = durationFromEnvironment("HCSSHIM_TIMEOUT_SYSCALLWATCHER", SyscallWatcher) - Tar2VHD = durationFromEnvironment("HCSSHIM_TIMEOUT_TAR2VHD", Tar2VHD) - ExternalCommandToStart = durationFromEnvironment("HCSSHIM_TIMEOUT_EXTERNALCOMMANDSTART", ExternalCommandToStart) - ExternalCommandToComplete = durationFromEnvironment("HCSSHIM_TIMEOUT_EXTERNALCOMMANDCOMPLETE", ExternalCommandToComplete) - TestDRetryLoop = durationFromEnvironment("HCSSHIM_TIMEOUT_TESTDRETRYLOOP", TestDRetryLoop) -} - -func durationFromEnvironment(env string, defaultValue time.Duration) time.Duration { - envTimeout := os.Getenv(env) - if len(envTimeout) > 0 { - e, err := strconv.Atoi(envTimeout) - if err == nil && e > 0 { - return time.Second * time.Duration(e) - } - } - return defaultValue -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/activatelayer.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/activatelayer.go deleted file mode 100644 index dcb919268..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/activatelayer.go +++ /dev/null @@ -1,32 +0,0 @@ -package wclayer - -import ( - "github.com/Microsoft/hcsshim/internal/hcserror" - "github.com/sirupsen/logrus" -) - -// ActivateLayer will find the layer with the given id and mount it's filesystem. -// For a read/write layer, the mounted filesystem will appear as a volume on the -// host, while a read-only layer is generally expected to be a no-op. -// An activated layer must later be deactivated via DeactivateLayer. -func ActivateLayer(path string) (err error) { - title := "hcsshim::ActivateLayer" - fields := logrus.Fields{ - "path": path, - } - logrus.WithFields(fields).Debug(title) - defer func() { - if err != nil { - fields[logrus.ErrorKey] = err - logrus.WithFields(fields).Error(err) - } else { - logrus.WithFields(fields).Debug(title + " - succeeded") - } - }() - - err = activateLayer(&stdDriverInfo, path) - if err != nil { - return hcserror.New(err, title+" - failed", "") - } - return nil -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/baselayer.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/baselayer.go deleted file mode 100644 index 5784241df..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/baselayer.go +++ /dev/null @@ -1,173 +0,0 @@ -package wclayer - -import ( - "errors" - "os" - "path/filepath" - "syscall" - - "github.com/Microsoft/go-winio" - "github.com/Microsoft/hcsshim/internal/hcserror" - "github.com/Microsoft/hcsshim/internal/safefile" -) - -type baseLayerWriter struct { - root *os.File - f *os.File - bw *winio.BackupFileWriter - err error - hasUtilityVM bool - dirInfo []dirInfo -} - -type dirInfo struct { - path string - fileInfo winio.FileBasicInfo -} - -// reapplyDirectoryTimes reapplies directory modification, creation, etc. times -// after processing of the directory tree has completed. The times are expected -// to be ordered such that parent directories come before child directories. -func reapplyDirectoryTimes(root *os.File, dis []dirInfo) error { - for i := range dis { - di := &dis[len(dis)-i-1] // reverse order: process child directories first - f, err := safefile.OpenRelative(di.path, root, syscall.GENERIC_READ|syscall.GENERIC_WRITE, syscall.FILE_SHARE_READ, safefile.FILE_OPEN, safefile.FILE_DIRECTORY_FILE) - if err != nil { - return err - } - - err = winio.SetFileBasicInfo(f, &di.fileInfo) - f.Close() - if err != nil { - return err - } - } - return nil -} - -func (w *baseLayerWriter) closeCurrentFile() error { - if w.f != nil { - err := w.bw.Close() - err2 := w.f.Close() - w.f = nil - w.bw = nil - if err != nil { - return err - } - if err2 != nil { - return err2 - } - } - return nil -} - -func (w *baseLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) (err error) { - defer func() { - if err != nil { - w.err = err - } - }() - - err = w.closeCurrentFile() - if err != nil { - return err - } - - if filepath.ToSlash(name) == `UtilityVM/Files` { - w.hasUtilityVM = true - } - - var f *os.File - defer func() { - if f != nil { - f.Close() - } - }() - - extraFlags := uint32(0) - if fileInfo.FileAttributes&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 { - extraFlags |= safefile.FILE_DIRECTORY_FILE - if fileInfo.FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT == 0 { - w.dirInfo = append(w.dirInfo, dirInfo{name, *fileInfo}) - } - } - - mode := uint32(syscall.GENERIC_READ | syscall.GENERIC_WRITE | winio.WRITE_DAC | winio.WRITE_OWNER | winio.ACCESS_SYSTEM_SECURITY) - f, err = safefile.OpenRelative(name, w.root, mode, syscall.FILE_SHARE_READ, safefile.FILE_CREATE, extraFlags) - if err != nil { - return hcserror.New(err, "Failed to safefile.OpenRelative", name) - } - - err = winio.SetFileBasicInfo(f, fileInfo) - if err != nil { - return hcserror.New(err, "Failed to SetFileBasicInfo", name) - } - - w.f = f - w.bw = winio.NewBackupFileWriter(f, true) - f = nil - return nil -} - -func (w *baseLayerWriter) AddLink(name string, target string) (err error) { - defer func() { - if err != nil { - w.err = err - } - }() - - err = w.closeCurrentFile() - if err != nil { - return err - } - - return safefile.LinkRelative(target, w.root, name, w.root) -} - -func (w *baseLayerWriter) Remove(name string) error { - return errors.New("base layer cannot have tombstones") -} - -func (w *baseLayerWriter) Write(b []byte) (int, error) { - n, err := w.bw.Write(b) - if err != nil { - w.err = err - } - return n, err -} - -func (w *baseLayerWriter) Close() error { - defer func() { - w.root.Close() - w.root = nil - }() - err := w.closeCurrentFile() - if err != nil { - return err - } - if w.err == nil { - // Restore the file times of all the directories, since they may have - // been modified by creating child directories. - err = reapplyDirectoryTimes(w.root, w.dirInfo) - if err != nil { - return err - } - - err = ProcessBaseLayer(w.root.Name()) - if err != nil { - return err - } - - if w.hasUtilityVM { - err := safefile.EnsureNotReparsePointRelative("UtilityVM", w.root) - if err != nil { - return err - } - err = ProcessUtilityVMImage(filepath.Join(w.root.Name(), "UtilityVM")) - if err != nil { - return err - } - } - } - return w.err -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/createlayer.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/createlayer.go deleted file mode 100644 index be2bc3fd6..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/createlayer.go +++ /dev/null @@ -1,31 +0,0 @@ -package wclayer - -import ( - "github.com/Microsoft/hcsshim/internal/hcserror" - "github.com/sirupsen/logrus" -) - -// CreateLayer creates a new, empty, read-only layer on the filesystem based on -// the parent layer provided. -func CreateLayer(path, parent string) (err error) { - title := "hcsshim::CreateLayer" - fields := logrus.Fields{ - "parent": parent, - "path": path, - } - logrus.WithFields(fields).Debug(title) - defer func() { - if err != nil { - fields[logrus.ErrorKey] = err - logrus.WithFields(fields).Error(err) - } else { - logrus.WithFields(fields).Debug(title + " - succeeded") - } - }() - - err = createLayer(&stdDriverInfo, path, parent) - if err != nil { - return hcserror.New(err, title+" - failed", "") - } - return nil -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/createscratchlayer.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/createscratchlayer.go deleted file mode 100644 index 7e3351289..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/createscratchlayer.go +++ /dev/null @@ -1,38 +0,0 @@ -package wclayer - -import ( - "github.com/Microsoft/hcsshim/internal/hcserror" - "github.com/sirupsen/logrus" -) - -// CreateScratchLayer creates and populates new read-write layer for use by a container. -// This requires both the id of the direct parent layer, as well as the full list -// of paths to all parent layers up to the base (and including the direct parent -// whose id was provided). -func CreateScratchLayer(path string, parentLayerPaths []string) (err error) { - title := "hcsshim::CreateScratchLayer" - fields := logrus.Fields{ - "path": path, - } - logrus.WithFields(fields).Debug(title) - defer func() { - if err != nil { - fields[logrus.ErrorKey] = err - logrus.WithFields(fields).Error(err) - } else { - logrus.WithFields(fields).Debug(title + " - succeeded") - } - }() - - // Generate layer descriptors - layers, err := layerPathsToDescriptors(parentLayerPaths) - if err != nil { - return err - } - - err = createSandboxLayer(&stdDriverInfo, path, 0, layers) - if err != nil { - return hcserror.New(err, title+" - failed", "") - } - return nil -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/deactivatelayer.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/deactivatelayer.go deleted file mode 100644 index 2dd5d5715..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/deactivatelayer.go +++ /dev/null @@ -1,29 +0,0 @@ -package wclayer - -import ( - "github.com/Microsoft/hcsshim/internal/hcserror" - "github.com/sirupsen/logrus" -) - -// DeactivateLayer will dismount a layer that was mounted via ActivateLayer. -func DeactivateLayer(path string) (err error) { - title := "hcsshim::DeactivateLayer" - fields := logrus.Fields{ - "path": path, - } - logrus.WithFields(fields).Debug(title) - defer func() { - if err != nil { - fields[logrus.ErrorKey] = err - logrus.WithFields(fields).Error(err) - } else { - logrus.WithFields(fields).Debug(title + " - succeeded") - } - }() - - err = deactivateLayer(&stdDriverInfo, path) - if err != nil { - return hcserror.New(err, title+"- failed", "") - } - return nil -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/destroylayer.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/destroylayer.go deleted file mode 100644 index 4da690c20..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/destroylayer.go +++ /dev/null @@ -1,30 +0,0 @@ -package wclayer - -import ( - "github.com/Microsoft/hcsshim/internal/hcserror" - "github.com/sirupsen/logrus" -) - -// DestroyLayer will remove the on-disk files representing the layer with the given -// path, including that layer's containing folder, if any. -func DestroyLayer(path string) (err error) { - title := "hcsshim::DestroyLayer" - fields := logrus.Fields{ - "path": path, - } - logrus.WithFields(fields).Debug(title) - defer func() { - if err != nil { - fields[logrus.ErrorKey] = err - logrus.WithFields(fields).Error(err) - } else { - logrus.WithFields(fields).Debug(title + " - succeeded") - } - }() - - err = destroyLayer(&stdDriverInfo, path) - if err != nil { - return hcserror.New(err, title+" - failed", "") - } - return nil -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/expandscratchsize.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/expandscratchsize.go deleted file mode 100644 index 651676fb2..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/expandscratchsize.go +++ /dev/null @@ -1,30 +0,0 @@ -package wclayer - -import ( - "github.com/Microsoft/hcsshim/internal/hcserror" - "github.com/sirupsen/logrus" -) - -// ExpandScratchSize expands the size of a layer to at least size bytes. -func ExpandScratchSize(path string, size uint64) (err error) { - title := "hcsshim::ExpandScratchSize" - fields := logrus.Fields{ - "path": path, - "size": size, - } - logrus.WithFields(fields).Debug(title) - defer func() { - if err != nil { - fields[logrus.ErrorKey] = err - logrus.WithFields(fields).Error(err) - } else { - logrus.WithFields(fields).Debug(title + " - succeeded") - } - }() - - err = expandSandboxSize(&stdDriverInfo, path, size) - if err != nil { - return hcserror.New(err, title+" - failed", "") - } - return nil -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/exportlayer.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/exportlayer.go deleted file mode 100644 index 0425b3395..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/exportlayer.go +++ /dev/null @@ -1,76 +0,0 @@ -package wclayer - -import ( - "io/ioutil" - "os" - - "github.com/Microsoft/go-winio" - "github.com/Microsoft/hcsshim/internal/hcserror" - "github.com/sirupsen/logrus" -) - -// ExportLayer will create a folder at exportFolderPath and fill that folder with -// the transport format version of the layer identified by layerId. This transport -// format includes any metadata required for later importing the layer (using -// ImportLayer), and requires the full list of parent layer paths in order to -// perform the export. -func ExportLayer(path string, exportFolderPath string, parentLayerPaths []string) (err error) { - title := "hcsshim::ExportLayer" - fields := logrus.Fields{ - "path": path, - "exportFolderPath": exportFolderPath, - } - logrus.WithFields(fields).Debug(title) - defer func() { - if err != nil { - fields[logrus.ErrorKey] = err - logrus.WithFields(fields).Error(err) - } else { - logrus.WithFields(fields).Debug(title + " - succeeded") - } - }() - - // Generate layer descriptors - layers, err := layerPathsToDescriptors(parentLayerPaths) - if err != nil { - return err - } - - err = exportLayer(&stdDriverInfo, path, exportFolderPath, layers) - if err != nil { - return hcserror.New(err, title+" - failed", "") - } - return nil -} - -type LayerReader interface { - Next() (string, int64, *winio.FileBasicInfo, error) - Read(b []byte) (int, error) - Close() error -} - -// NewLayerReader returns a new layer reader for reading the contents of an on-disk layer. -// The caller must have taken the SeBackupPrivilege privilege -// to call this and any methods on the resulting LayerReader. -func NewLayerReader(path string, parentLayerPaths []string) (LayerReader, error) { - exportPath, err := ioutil.TempDir("", "hcs") - if err != nil { - return nil, err - } - err = ExportLayer(path, exportPath, parentLayerPaths) - if err != nil { - os.RemoveAll(exportPath) - return nil, err - } - return &legacyLayerReaderWrapper{newLegacyLayerReader(exportPath)}, nil -} - -type legacyLayerReaderWrapper struct { - *legacyLayerReader -} - -func (r *legacyLayerReaderWrapper) Close() error { - err := r.legacyLayerReader.Close() - os.RemoveAll(r.root) - return err -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/getlayermountpath.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/getlayermountpath.go deleted file mode 100644 index d60b6ed53..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/getlayermountpath.go +++ /dev/null @@ -1,56 +0,0 @@ -package wclayer - -import ( - "syscall" - - "github.com/Microsoft/hcsshim/internal/hcserror" - "github.com/sirupsen/logrus" -) - -// GetLayerMountPath will look for a mounted layer with the given path and return -// the path at which that layer can be accessed. This path may be a volume path -// if the layer is a mounted read-write layer, otherwise it is expected to be the -// folder path at which the layer is stored. -func GetLayerMountPath(path string) (_ string, err error) { - title := "hcsshim::GetLayerMountPath" - fields := logrus.Fields{ - "path": path, - } - logrus.WithFields(fields).Debug(title) - defer func() { - if err != nil { - fields[logrus.ErrorKey] = err - logrus.WithFields(fields).Error(err) - } else { - logrus.WithFields(fields).Debug(title + " - succeeded") - } - }() - - var mountPathLength uintptr - mountPathLength = 0 - - // Call the procedure itself. - logrus.WithFields(fields).Debug("Calling proc (1)") - err = getLayerMountPath(&stdDriverInfo, path, &mountPathLength, nil) - if err != nil { - return "", hcserror.New(err, title+" - failed", "(first call)") - } - - // Allocate a mount path of the returned length. - if mountPathLength == 0 { - return "", nil - } - mountPathp := make([]uint16, mountPathLength) - mountPathp[0] = 0 - - // Call the procedure again - logrus.WithFields(fields).Debug("Calling proc (2)") - err = getLayerMountPath(&stdDriverInfo, path, &mountPathLength, &mountPathp[0]) - if err != nil { - return "", hcserror.New(err, title+" - failed", "(second call)") - } - - mountPath := syscall.UTF16ToString(mountPathp[0:]) - fields["mountPath"] = mountPath - return mountPath, nil -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/getsharedbaseimages.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/getsharedbaseimages.go deleted file mode 100644 index dbd83ef2b..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/getsharedbaseimages.go +++ /dev/null @@ -1,29 +0,0 @@ -package wclayer - -import ( - "github.com/Microsoft/hcsshim/internal/hcserror" - "github.com/Microsoft/hcsshim/internal/interop" - "github.com/sirupsen/logrus" -) - -// GetSharedBaseImages will enumerate the images stored in the common central -// image store and return descriptive info about those images for the purpose -// of registering them with the graphdriver, graph, and tagstore. -func GetSharedBaseImages() (imageData string, err error) { - title := "hcsshim::GetSharedBaseImages" - logrus.Debug(title) - defer func() { - if err != nil { - logrus.WithError(err).Error(err) - } else { - logrus.WithField("imageData", imageData).Debug(title + " - succeeded") - } - }() - - var buffer *uint16 - err = getBaseImages(&buffer) - if err != nil { - return "", hcserror.New(err, title+" - failed", "") - } - return interop.ConvertAndFreeCoTaskMemString(buffer), nil -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/grantvmaccess.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/grantvmaccess.go deleted file mode 100644 index 05735df6c..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/grantvmaccess.go +++ /dev/null @@ -1,30 +0,0 @@ -package wclayer - -import ( - "github.com/Microsoft/hcsshim/internal/hcserror" - "github.com/sirupsen/logrus" -) - -// GrantVmAccess adds access to a file for a given VM -func GrantVmAccess(vmid string, filepath string) (err error) { - title := "hcsshim::GrantVmAccess" - fields := logrus.Fields{ - "vm-id": vmid, - "path": filepath, - } - logrus.WithFields(fields).Debug(title) - defer func() { - if err != nil { - fields[logrus.ErrorKey] = err - logrus.WithFields(fields).Error(err) - } else { - logrus.WithFields(fields).Debug(title + " - succeeded") - } - }() - - err = grantVmAccess(vmid, filepath) - if err != nil { - return hcserror.New(err, title+" - failed", "") - } - return nil -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/importlayer.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/importlayer.go deleted file mode 100644 index 76a804f2a..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/importlayer.go +++ /dev/null @@ -1,135 +0,0 @@ -package wclayer - -import ( - "io/ioutil" - "os" - "path/filepath" - - "github.com/Microsoft/go-winio" - "github.com/Microsoft/hcsshim/internal/hcserror" - "github.com/Microsoft/hcsshim/internal/safefile" - "github.com/sirupsen/logrus" -) - -// ImportLayer will take the contents of the folder at importFolderPath and import -// that into a layer with the id layerId. Note that in order to correctly populate -// the layer and interperet the transport format, all parent layers must already -// be present on the system at the paths provided in parentLayerPaths. -func ImportLayer(path string, importFolderPath string, parentLayerPaths []string) (err error) { - title := "hcsshim::ImportLayer" - fields := logrus.Fields{ - "path": path, - "importFolderPath": importFolderPath, - } - logrus.WithFields(fields).Debug(title) - defer func() { - if err != nil { - fields[logrus.ErrorKey] = err - logrus.WithFields(fields).Error(err) - } else { - logrus.WithFields(fields).Debug(title + " - succeeded") - } - }() - - // Generate layer descriptors - layers, err := layerPathsToDescriptors(parentLayerPaths) - if err != nil { - return err - } - - err = importLayer(&stdDriverInfo, path, importFolderPath, layers) - if err != nil { - return hcserror.New(err, title+" - failed", "") - } - return nil -} - -// LayerWriter is an interface that supports writing a new container image layer. -type LayerWriter interface { - // Add adds a file to the layer with given metadata. - Add(name string, fileInfo *winio.FileBasicInfo) error - // AddLink adds a hard link to the layer. The target must already have been added. - AddLink(name string, target string) error - // Remove removes a file that was present in a parent layer from the layer. - Remove(name string) error - // Write writes data to the current file. The data must be in the format of a Win32 - // backup stream. - Write(b []byte) (int, error) - // Close finishes the layer writing process and releases any resources. - Close() error -} - -type legacyLayerWriterWrapper struct { - *legacyLayerWriter - path string - parentLayerPaths []string -} - -func (r *legacyLayerWriterWrapper) Close() error { - defer os.RemoveAll(r.root.Name()) - defer r.legacyLayerWriter.CloseRoots() - err := r.legacyLayerWriter.Close() - if err != nil { - return err - } - - if err = ImportLayer(r.destRoot.Name(), r.path, r.parentLayerPaths); err != nil { - return err - } - for _, name := range r.Tombstones { - if err = safefile.RemoveRelative(name, r.destRoot); err != nil && !os.IsNotExist(err) { - return err - } - } - // Add any hard links that were collected. - for _, lnk := range r.PendingLinks { - if err = safefile.RemoveRelative(lnk.Path, r.destRoot); err != nil && !os.IsNotExist(err) { - return err - } - if err = safefile.LinkRelative(lnk.Target, lnk.TargetRoot, lnk.Path, r.destRoot); err != nil { - return err - } - } - // Prepare the utility VM for use if one is present in the layer. - if r.HasUtilityVM { - err := safefile.EnsureNotReparsePointRelative("UtilityVM", r.destRoot) - if err != nil { - return err - } - err = ProcessUtilityVMImage(filepath.Join(r.destRoot.Name(), "UtilityVM")) - if err != nil { - return err - } - } - return nil -} - -// NewLayerWriter returns a new layer writer for creating a layer on disk. -// The caller must have taken the SeBackupPrivilege and SeRestorePrivilege privileges -// to call this and any methods on the resulting LayerWriter. -func NewLayerWriter(path string, parentLayerPaths []string) (LayerWriter, error) { - if len(parentLayerPaths) == 0 { - // This is a base layer. It gets imported differently. - f, err := safefile.OpenRoot(path) - if err != nil { - return nil, err - } - return &baseLayerWriter{ - root: f, - }, nil - } - - importPath, err := ioutil.TempDir("", "hcs") - if err != nil { - return nil, err - } - w, err := newLegacyLayerWriter(importPath, parentLayerPaths, path) - if err != nil { - return nil, err - } - return &legacyLayerWriterWrapper{ - legacyLayerWriter: w, - path: importPath, - parentLayerPaths: parentLayerPaths, - }, nil -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/layerexists.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/layerexists.go deleted file mode 100644 index 258167a57..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/layerexists.go +++ /dev/null @@ -1,33 +0,0 @@ -package wclayer - -import ( - "github.com/Microsoft/hcsshim/internal/hcserror" - "github.com/sirupsen/logrus" -) - -// LayerExists will return true if a layer with the given id exists and is known -// to the system. -func LayerExists(path string) (_ bool, err error) { - title := "hcsshim::LayerExists" - fields := logrus.Fields{ - "path": path, - } - logrus.WithFields(fields).Debug(title) - defer func() { - if err != nil { - fields[logrus.ErrorKey] = err - logrus.WithFields(fields).Error(err) - } else { - logrus.WithFields(fields).Debug(title + " - succeeded") - } - }() - - // Call the procedure itself. - var exists uint32 - err = layerExists(&stdDriverInfo, path, &exists) - if err != nil { - return false, hcserror.New(err, title+" - failed", "") - } - fields["layer-exists"] = exists != 0 - return exists != 0, nil -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/layerid.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/layerid.go deleted file mode 100644 index 90df3bedc..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/layerid.go +++ /dev/null @@ -1,13 +0,0 @@ -package wclayer - -import ( - "path/filepath" - - "github.com/Microsoft/hcsshim/internal/guid" -) - -// LayerID returns the layer ID of a layer on disk. -func LayerID(path string) (guid.GUID, error) { - _, file := filepath.Split(path) - return NameToGuid(file) -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/layerutils.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/layerutils.go deleted file mode 100644 index 6d0ae8a07..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/layerutils.go +++ /dev/null @@ -1,96 +0,0 @@ -package wclayer - -// This file contains utility functions to support storage (graph) related -// functionality. - -import ( - "syscall" - - "github.com/Microsoft/hcsshim/internal/guid" - "github.com/sirupsen/logrus" -) - -/* To pass into syscall, we need a struct matching the following: -enum GraphDriverType -{ - DiffDriver, - FilterDriver -}; - -struct DriverInfo { - GraphDriverType Flavour; - LPCWSTR HomeDir; -}; -*/ - -type driverInfo struct { - Flavour int - HomeDirp *uint16 -} - -var ( - utf16EmptyString uint16 - stdDriverInfo = driverInfo{1, &utf16EmptyString} -) - -/* To pass into syscall, we need a struct matching the following: -typedef struct _WC_LAYER_DESCRIPTOR { - - // - // The ID of the layer - // - - GUID LayerId; - - // - // Additional flags - // - - union { - struct { - ULONG Reserved : 31; - ULONG Dirty : 1; // Created from sandbox as a result of snapshot - }; - ULONG Value; - } Flags; - - // - // Path to the layer root directory, null-terminated - // - - PCWSTR Path; - -} WC_LAYER_DESCRIPTOR, *PWC_LAYER_DESCRIPTOR; -*/ -type WC_LAYER_DESCRIPTOR struct { - LayerId guid.GUID - Flags uint32 - Pathp *uint16 -} - -func layerPathsToDescriptors(parentLayerPaths []string) ([]WC_LAYER_DESCRIPTOR, error) { - // Array of descriptors that gets constructed. - var layers []WC_LAYER_DESCRIPTOR - - for i := 0; i < len(parentLayerPaths); i++ { - g, err := LayerID(parentLayerPaths[i]) - if err != nil { - logrus.WithError(err).Debug("Failed to convert name to guid") - return nil, err - } - - p, err := syscall.UTF16PtrFromString(parentLayerPaths[i]) - if err != nil { - logrus.WithError(err).Debug("Failed conversion of parentLayerPath to pointer") - return nil, err - } - - layers = append(layers, WC_LAYER_DESCRIPTOR{ - LayerId: g, - Flags: 0, - Pathp: p, - }) - } - - return layers, nil -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/legacy.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/legacy.go deleted file mode 100644 index b8ea5d263..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/legacy.go +++ /dev/null @@ -1,815 +0,0 @@ -package wclayer - -import ( - "bufio" - "encoding/binary" - "errors" - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - "strings" - "syscall" - - "github.com/Microsoft/go-winio" - "github.com/Microsoft/hcsshim/internal/longpath" - "github.com/Microsoft/hcsshim/internal/safefile" -) - -var errorIterationCanceled = errors.New("") - -var mutatedUtilityVMFiles = map[string]bool{ - `EFI\Microsoft\Boot\BCD`: true, - `EFI\Microsoft\Boot\BCD.LOG`: true, - `EFI\Microsoft\Boot\BCD.LOG1`: true, - `EFI\Microsoft\Boot\BCD.LOG2`: true, -} - -const ( - filesPath = `Files` - hivesPath = `Hives` - utilityVMPath = `UtilityVM` - utilityVMFilesPath = `UtilityVM\Files` -) - -func openFileOrDir(path string, mode uint32, createDisposition uint32) (file *os.File, err error) { - return winio.OpenForBackup(path, mode, syscall.FILE_SHARE_READ, createDisposition) -} - -func hasPathPrefix(p, prefix string) bool { - return strings.HasPrefix(p, prefix) && len(p) > len(prefix) && p[len(prefix)] == '\\' -} - -type fileEntry struct { - path string - fi os.FileInfo - err error -} - -type legacyLayerReader struct { - root string - result chan *fileEntry - proceed chan bool - currentFile *os.File - backupReader *winio.BackupFileReader -} - -// newLegacyLayerReader returns a new LayerReader that can read the Windows -// container layer transport format from disk. -func newLegacyLayerReader(root string) *legacyLayerReader { - r := &legacyLayerReader{ - root: root, - result: make(chan *fileEntry), - proceed: make(chan bool), - } - go r.walk() - return r -} - -func readTombstones(path string) (map[string]([]string), error) { - tf, err := os.Open(filepath.Join(path, "tombstones.txt")) - if err != nil { - return nil, err - } - defer tf.Close() - s := bufio.NewScanner(tf) - if !s.Scan() || s.Text() != "\xef\xbb\xbfVersion 1.0" { - return nil, errors.New("Invalid tombstones file") - } - - ts := make(map[string]([]string)) - for s.Scan() { - t := filepath.Join(filesPath, s.Text()[1:]) // skip leading `\` - dir := filepath.Dir(t) - ts[dir] = append(ts[dir], t) - } - if err = s.Err(); err != nil { - return nil, err - } - - return ts, nil -} - -func (r *legacyLayerReader) walkUntilCancelled() error { - root, err := longpath.LongAbs(r.root) - if err != nil { - return err - } - - r.root = root - ts, err := readTombstones(r.root) - if err != nil { - return err - } - - err = filepath.Walk(r.root, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - // Indirect fix for https://github.com/moby/moby/issues/32838#issuecomment-343610048. - // Handle failure from what may be a golang bug in the conversion of - // UTF16 to UTF8 in files which are left in the recycle bin. Os.Lstat - // which is called by filepath.Walk will fail when a filename contains - // unicode characters. Skip the recycle bin regardless which is goodness. - if strings.EqualFold(path, filepath.Join(r.root, `Files\$Recycle.Bin`)) && info.IsDir() { - return filepath.SkipDir - } - - if path == r.root || path == filepath.Join(r.root, "tombstones.txt") || strings.HasSuffix(path, ".$wcidirs$") { - return nil - } - - r.result <- &fileEntry{path, info, nil} - if !<-r.proceed { - return errorIterationCanceled - } - - // List all the tombstones. - if info.IsDir() { - relPath, err := filepath.Rel(r.root, path) - if err != nil { - return err - } - if dts, ok := ts[relPath]; ok { - for _, t := range dts { - r.result <- &fileEntry{filepath.Join(r.root, t), nil, nil} - if !<-r.proceed { - return errorIterationCanceled - } - } - } - } - return nil - }) - if err == errorIterationCanceled { - return nil - } - if err == nil { - return io.EOF - } - return err -} - -func (r *legacyLayerReader) walk() { - defer close(r.result) - if !<-r.proceed { - return - } - - err := r.walkUntilCancelled() - if err != nil { - for { - r.result <- &fileEntry{err: err} - if !<-r.proceed { - return - } - } - } -} - -func (r *legacyLayerReader) reset() { - if r.backupReader != nil { - r.backupReader.Close() - r.backupReader = nil - } - if r.currentFile != nil { - r.currentFile.Close() - r.currentFile = nil - } -} - -func findBackupStreamSize(r io.Reader) (int64, error) { - br := winio.NewBackupStreamReader(r) - for { - hdr, err := br.Next() - if err != nil { - if err == io.EOF { - err = nil - } - return 0, err - } - if hdr.Id == winio.BackupData { - return hdr.Size, nil - } - } -} - -func (r *legacyLayerReader) Next() (path string, size int64, fileInfo *winio.FileBasicInfo, err error) { - r.reset() - r.proceed <- true - fe := <-r.result - if fe == nil { - err = errors.New("LegacyLayerReader closed") - return - } - if fe.err != nil { - err = fe.err - return - } - - path, err = filepath.Rel(r.root, fe.path) - if err != nil { - return - } - - if fe.fi == nil { - // This is a tombstone. Return a nil fileInfo. - return - } - - if fe.fi.IsDir() && hasPathPrefix(path, filesPath) { - fe.path += ".$wcidirs$" - } - - f, err := openFileOrDir(fe.path, syscall.GENERIC_READ, syscall.OPEN_EXISTING) - if err != nil { - return - } - defer func() { - if f != nil { - f.Close() - } - }() - - fileInfo, err = winio.GetFileBasicInfo(f) - if err != nil { - return - } - - if !hasPathPrefix(path, filesPath) { - size = fe.fi.Size() - r.backupReader = winio.NewBackupFileReader(f, false) - if path == hivesPath || path == filesPath { - // The Hives directory has a non-deterministic file time because of the - // nature of the import process. Use the times from System_Delta. - var g *os.File - g, err = os.Open(filepath.Join(r.root, hivesPath, `System_Delta`)) - if err != nil { - return - } - attr := fileInfo.FileAttributes - fileInfo, err = winio.GetFileBasicInfo(g) - g.Close() - if err != nil { - return - } - fileInfo.FileAttributes = attr - } - - // The creation time and access time get reset for files outside of the Files path. - fileInfo.CreationTime = fileInfo.LastWriteTime - fileInfo.LastAccessTime = fileInfo.LastWriteTime - - } else { - // The file attributes are written before the backup stream. - var attr uint32 - err = binary.Read(f, binary.LittleEndian, &attr) - if err != nil { - return - } - fileInfo.FileAttributes = attr - beginning := int64(4) - - // Find the accurate file size. - if !fe.fi.IsDir() { - size, err = findBackupStreamSize(f) - if err != nil { - err = &os.PathError{Op: "findBackupStreamSize", Path: fe.path, Err: err} - return - } - } - - // Return back to the beginning of the backup stream. - _, err = f.Seek(beginning, 0) - if err != nil { - return - } - } - - r.currentFile = f - f = nil - return -} - -func (r *legacyLayerReader) Read(b []byte) (int, error) { - if r.backupReader == nil { - if r.currentFile == nil { - return 0, io.EOF - } - return r.currentFile.Read(b) - } - return r.backupReader.Read(b) -} - -func (r *legacyLayerReader) Seek(offset int64, whence int) (int64, error) { - if r.backupReader == nil { - if r.currentFile == nil { - return 0, errors.New("no current file") - } - return r.currentFile.Seek(offset, whence) - } - return 0, errors.New("seek not supported on this stream") -} - -func (r *legacyLayerReader) Close() error { - r.proceed <- false - <-r.result - r.reset() - return nil -} - -type pendingLink struct { - Path, Target string - TargetRoot *os.File -} - -type pendingDir struct { - Path string - Root *os.File -} - -type legacyLayerWriter struct { - root *os.File - destRoot *os.File - parentRoots []*os.File - currentFile *os.File - bufWriter *bufio.Writer - currentFileName string - currentFileRoot *os.File - backupWriter *winio.BackupFileWriter - Tombstones []string - HasUtilityVM bool - uvmDi []dirInfo - addedFiles map[string]bool - PendingLinks []pendingLink - pendingDirs []pendingDir - currentIsDir bool -} - -// newLegacyLayerWriter returns a LayerWriter that can write the contaler layer -// transport format to disk. -func newLegacyLayerWriter(root string, parentRoots []string, destRoot string) (w *legacyLayerWriter, err error) { - w = &legacyLayerWriter{ - addedFiles: make(map[string]bool), - } - defer func() { - if err != nil { - w.CloseRoots() - w = nil - } - }() - w.root, err = safefile.OpenRoot(root) - if err != nil { - return - } - w.destRoot, err = safefile.OpenRoot(destRoot) - if err != nil { - return - } - for _, r := range parentRoots { - f, err := safefile.OpenRoot(r) - if err != nil { - return w, err - } - w.parentRoots = append(w.parentRoots, f) - } - w.bufWriter = bufio.NewWriterSize(ioutil.Discard, 65536) - return -} - -func (w *legacyLayerWriter) CloseRoots() { - if w.root != nil { - w.root.Close() - w.root = nil - } - if w.destRoot != nil { - w.destRoot.Close() - w.destRoot = nil - } - for i := range w.parentRoots { - w.parentRoots[i].Close() - } - w.parentRoots = nil -} - -func (w *legacyLayerWriter) initUtilityVM() error { - if !w.HasUtilityVM { - err := safefile.MkdirRelative(utilityVMPath, w.destRoot) - if err != nil { - return err - } - // Server 2016 does not support multiple layers for the utility VM, so - // clone the utility VM from the parent layer into this layer. Use hard - // links to avoid unnecessary copying, since most of the files are - // immutable. - err = cloneTree(w.parentRoots[0], w.destRoot, utilityVMFilesPath, mutatedUtilityVMFiles) - if err != nil { - return fmt.Errorf("cloning the parent utility VM image failed: %s", err) - } - w.HasUtilityVM = true - } - return nil -} - -func (w *legacyLayerWriter) reset() error { - err := w.bufWriter.Flush() - if err != nil { - return err - } - w.bufWriter.Reset(ioutil.Discard) - if w.currentIsDir { - r := w.currentFile - br := winio.NewBackupStreamReader(r) - // Seek to the beginning of the backup stream, skipping the fileattrs - if _, err := r.Seek(4, io.SeekStart); err != nil { - return err - } - - for { - bhdr, err := br.Next() - if err == io.EOF { - // end of backupstream data - break - } - if err != nil { - return err - } - switch bhdr.Id { - case winio.BackupReparseData: - // The current file is a `.$wcidirs$` metadata file that - // describes a directory reparse point. Delete the placeholder - // directory to prevent future files being added into the - // destination of the reparse point during the ImportLayer call - if err := safefile.RemoveRelative(w.currentFileName, w.currentFileRoot); err != nil { - return err - } - w.pendingDirs = append(w.pendingDirs, pendingDir{Path: w.currentFileName, Root: w.currentFileRoot}) - default: - // ignore all other stream types, as we only care about directory reparse points - } - } - w.currentIsDir = false - } - if w.backupWriter != nil { - w.backupWriter.Close() - w.backupWriter = nil - } - if w.currentFile != nil { - w.currentFile.Close() - w.currentFile = nil - w.currentFileName = "" - w.currentFileRoot = nil - } - return nil -} - -// copyFileWithMetadata copies a file using the backup/restore APIs in order to preserve metadata -func copyFileWithMetadata(srcRoot, destRoot *os.File, subPath string, isDir bool) (fileInfo *winio.FileBasicInfo, err error) { - src, err := safefile.OpenRelative( - subPath, - srcRoot, - syscall.GENERIC_READ|winio.ACCESS_SYSTEM_SECURITY, - syscall.FILE_SHARE_READ, - safefile.FILE_OPEN, - safefile.FILE_OPEN_REPARSE_POINT) - if err != nil { - return nil, err - } - defer src.Close() - srcr := winio.NewBackupFileReader(src, true) - defer srcr.Close() - - fileInfo, err = winio.GetFileBasicInfo(src) - if err != nil { - return nil, err - } - - extraFlags := uint32(0) - if isDir { - extraFlags |= safefile.FILE_DIRECTORY_FILE - } - dest, err := safefile.OpenRelative( - subPath, - destRoot, - syscall.GENERIC_READ|syscall.GENERIC_WRITE|winio.WRITE_DAC|winio.WRITE_OWNER|winio.ACCESS_SYSTEM_SECURITY, - syscall.FILE_SHARE_READ, - safefile.FILE_CREATE, - extraFlags) - if err != nil { - return nil, err - } - defer dest.Close() - - err = winio.SetFileBasicInfo(dest, fileInfo) - if err != nil { - return nil, err - } - - destw := winio.NewBackupFileWriter(dest, true) - defer func() { - cerr := destw.Close() - if err == nil { - err = cerr - } - }() - - _, err = io.Copy(destw, srcr) - if err != nil { - return nil, err - } - - return fileInfo, nil -} - -// cloneTree clones a directory tree using hard links. It skips hard links for -// the file names in the provided map and just copies those files. -func cloneTree(srcRoot *os.File, destRoot *os.File, subPath string, mutatedFiles map[string]bool) error { - var di []dirInfo - err := safefile.EnsureNotReparsePointRelative(subPath, srcRoot) - if err != nil { - return err - } - err = filepath.Walk(filepath.Join(srcRoot.Name(), subPath), func(srcFilePath string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - relPath, err := filepath.Rel(srcRoot.Name(), srcFilePath) - if err != nil { - return err - } - - fileAttributes := info.Sys().(*syscall.Win32FileAttributeData).FileAttributes - // Directories, reparse points, and files that will be mutated during - // utility VM import must be copied. All other files can be hard linked. - isReparsePoint := fileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT != 0 - // In go1.9, FileInfo.IsDir() returns false if the directory is also a symlink. - // See: https://github.com/golang/go/commit/1989921aef60c83e6f9127a8448fb5ede10e9acc - // Fixes the problem by checking syscall.FILE_ATTRIBUTE_DIRECTORY directly - isDir := fileAttributes&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 - - if isDir || isReparsePoint || mutatedFiles[relPath] { - fi, err := copyFileWithMetadata(srcRoot, destRoot, relPath, isDir) - if err != nil { - return err - } - if isDir && !isReparsePoint { - di = append(di, dirInfo{path: relPath, fileInfo: *fi}) - } - } else { - err = safefile.LinkRelative(relPath, srcRoot, relPath, destRoot) - if err != nil { - return err - } - } - - return nil - }) - if err != nil { - return err - } - - return reapplyDirectoryTimes(destRoot, di) -} - -func (w *legacyLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) error { - if err := w.reset(); err != nil { - return err - } - - if name == utilityVMPath { - return w.initUtilityVM() - } - - name = filepath.Clean(name) - if hasPathPrefix(name, utilityVMPath) { - if !w.HasUtilityVM { - return errors.New("missing UtilityVM directory") - } - if !hasPathPrefix(name, utilityVMFilesPath) && name != utilityVMFilesPath { - return errors.New("invalid UtilityVM layer") - } - createDisposition := uint32(safefile.FILE_OPEN) - if (fileInfo.FileAttributes & syscall.FILE_ATTRIBUTE_DIRECTORY) != 0 { - st, err := safefile.LstatRelative(name, w.destRoot) - if err != nil && !os.IsNotExist(err) { - return err - } - if st != nil { - // Delete the existing file/directory if it is not the same type as this directory. - existingAttr := st.Sys().(*syscall.Win32FileAttributeData).FileAttributes - if (uint32(fileInfo.FileAttributes)^existingAttr)&(syscall.FILE_ATTRIBUTE_DIRECTORY|syscall.FILE_ATTRIBUTE_REPARSE_POINT) != 0 { - if err = safefile.RemoveAllRelative(name, w.destRoot); err != nil { - return err - } - st = nil - } - } - if st == nil { - if err = safefile.MkdirRelative(name, w.destRoot); err != nil { - return err - } - } - if fileInfo.FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT == 0 { - w.uvmDi = append(w.uvmDi, dirInfo{path: name, fileInfo: *fileInfo}) - } - } else { - // Overwrite any existing hard link. - err := safefile.RemoveRelative(name, w.destRoot) - if err != nil && !os.IsNotExist(err) { - return err - } - createDisposition = safefile.FILE_CREATE - } - - f, err := safefile.OpenRelative( - name, - w.destRoot, - syscall.GENERIC_READ|syscall.GENERIC_WRITE|winio.WRITE_DAC|winio.WRITE_OWNER|winio.ACCESS_SYSTEM_SECURITY, - syscall.FILE_SHARE_READ, - createDisposition, - safefile.FILE_OPEN_REPARSE_POINT, - ) - if err != nil { - return err - } - defer func() { - if f != nil { - f.Close() - safefile.RemoveRelative(name, w.destRoot) - } - }() - - err = winio.SetFileBasicInfo(f, fileInfo) - if err != nil { - return err - } - - w.backupWriter = winio.NewBackupFileWriter(f, true) - w.bufWriter.Reset(w.backupWriter) - w.currentFile = f - w.currentFileName = name - w.currentFileRoot = w.destRoot - w.addedFiles[name] = true - f = nil - return nil - } - - fname := name - if (fileInfo.FileAttributes & syscall.FILE_ATTRIBUTE_DIRECTORY) != 0 { - err := safefile.MkdirRelative(name, w.root) - if err != nil { - return err - } - fname += ".$wcidirs$" - w.currentIsDir = true - } - - f, err := safefile.OpenRelative(fname, w.root, syscall.GENERIC_READ|syscall.GENERIC_WRITE, syscall.FILE_SHARE_READ, safefile.FILE_CREATE, 0) - if err != nil { - return err - } - defer func() { - if f != nil { - f.Close() - safefile.RemoveRelative(fname, w.root) - } - }() - - strippedFi := *fileInfo - strippedFi.FileAttributes = 0 - err = winio.SetFileBasicInfo(f, &strippedFi) - if err != nil { - return err - } - - if hasPathPrefix(name, hivesPath) { - w.backupWriter = winio.NewBackupFileWriter(f, false) - w.bufWriter.Reset(w.backupWriter) - } else { - w.bufWriter.Reset(f) - // The file attributes are written before the stream. - err = binary.Write(w.bufWriter, binary.LittleEndian, uint32(fileInfo.FileAttributes)) - if err != nil { - w.bufWriter.Reset(ioutil.Discard) - return err - } - } - - w.currentFile = f - w.currentFileName = name - w.currentFileRoot = w.root - w.addedFiles[name] = true - f = nil - return nil -} - -func (w *legacyLayerWriter) AddLink(name string, target string) error { - if err := w.reset(); err != nil { - return err - } - - target = filepath.Clean(target) - var roots []*os.File - if hasPathPrefix(target, filesPath) { - // Look for cross-layer hard link targets in the parent layers, since - // nothing is in the destination path yet. - roots = w.parentRoots - } else if hasPathPrefix(target, utilityVMFilesPath) { - // Since the utility VM is fully cloned into the destination path - // already, look for cross-layer hard link targets directly in the - // destination path. - roots = []*os.File{w.destRoot} - } - - if roots == nil || (!hasPathPrefix(name, filesPath) && !hasPathPrefix(name, utilityVMFilesPath)) { - return errors.New("invalid hard link in layer") - } - - // Find to try the target of the link in a previously added file. If that - // fails, search in parent layers. - var selectedRoot *os.File - if _, ok := w.addedFiles[target]; ok { - selectedRoot = w.destRoot - } else { - for _, r := range roots { - if _, err := safefile.LstatRelative(target, r); err != nil { - if !os.IsNotExist(err) { - return err - } - } else { - selectedRoot = r - break - } - } - if selectedRoot == nil { - return fmt.Errorf("failed to find link target for '%s' -> '%s'", name, target) - } - } - - // The link can't be written until after the ImportLayer call. - w.PendingLinks = append(w.PendingLinks, pendingLink{ - Path: name, - Target: target, - TargetRoot: selectedRoot, - }) - w.addedFiles[name] = true - return nil -} - -func (w *legacyLayerWriter) Remove(name string) error { - name = filepath.Clean(name) - if hasPathPrefix(name, filesPath) { - w.Tombstones = append(w.Tombstones, name) - } else if hasPathPrefix(name, utilityVMFilesPath) { - err := w.initUtilityVM() - if err != nil { - return err - } - // Make sure the path exists; os.RemoveAll will not fail if the file is - // already gone, and this needs to be a fatal error for diagnostics - // purposes. - if _, err := safefile.LstatRelative(name, w.destRoot); err != nil { - return err - } - err = safefile.RemoveAllRelative(name, w.destRoot) - if err != nil { - return err - } - } else { - return fmt.Errorf("invalid tombstone %s", name) - } - - return nil -} - -func (w *legacyLayerWriter) Write(b []byte) (int, error) { - if w.backupWriter == nil && w.currentFile == nil { - return 0, errors.New("closed") - } - return w.bufWriter.Write(b) -} - -func (w *legacyLayerWriter) Close() error { - if err := w.reset(); err != nil { - return err - } - if err := safefile.RemoveRelative("tombstones.txt", w.root); err != nil && !os.IsNotExist(err) { - return err - } - for _, pd := range w.pendingDirs { - err := safefile.MkdirRelative(pd.Path, pd.Root) - if err != nil { - return err - } - } - if w.HasUtilityVM { - err := reapplyDirectoryTimes(w.destRoot, w.uvmDi) - if err != nil { - return err - } - } - return nil -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/nametoguid.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/nametoguid.go deleted file mode 100644 index 45a63cf65..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/nametoguid.go +++ /dev/null @@ -1,34 +0,0 @@ -package wclayer - -import ( - "github.com/Microsoft/hcsshim/internal/guid" - "github.com/Microsoft/hcsshim/internal/hcserror" - "github.com/sirupsen/logrus" -) - -// NameToGuid converts the given string into a GUID using the algorithm in the -// Host Compute Service, ensuring GUIDs generated with the same string are common -// across all clients. -func NameToGuid(name string) (id guid.GUID, err error) { - title := "hcsshim::NameToGuid" - fields := logrus.Fields{ - "name": name, - } - logrus.WithFields(fields).Debug(title) - defer func() { - if err != nil { - fields[logrus.ErrorKey] = err - logrus.WithFields(fields).Error(err) - } else { - logrus.WithFields(fields).Debug(title + " - succeeded") - } - }() - - err = nameToGuid(name, &id) - if err != nil { - err = hcserror.New(err, title+" - failed", "") - return - } - fields["guid"] = id.String() - return -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/preparelayer.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/preparelayer.go deleted file mode 100644 index 2b65b0186..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/preparelayer.go +++ /dev/null @@ -1,47 +0,0 @@ -package wclayer - -import ( - "sync" - - "github.com/Microsoft/hcsshim/internal/hcserror" - "github.com/sirupsen/logrus" -) - -var prepareLayerLock sync.Mutex - -// PrepareLayer finds a mounted read-write layer matching path and enables the -// the filesystem filter for use on that layer. This requires the paths to all -// parent layers, and is necessary in order to view or interact with the layer -// as an actual filesystem (reading and writing files, creating directories, etc). -// Disabling the filter must be done via UnprepareLayer. -func PrepareLayer(path string, parentLayerPaths []string) (err error) { - title := "hcsshim::PrepareLayer" - fields := logrus.Fields{ - "path": path, - } - logrus.WithFields(fields).Debug(title) - defer func() { - if err != nil { - fields[logrus.ErrorKey] = err - logrus.WithFields(fields).Error(err) - } else { - logrus.WithFields(fields).Debug(title + " - succeeded") - } - }() - - // Generate layer descriptors - layers, err := layerPathsToDescriptors(parentLayerPaths) - if err != nil { - return err - } - - // This lock is a temporary workaround for a Windows bug. Only allowing one - // call to prepareLayer at a time vastly reduces the chance of a timeout. - prepareLayerLock.Lock() - defer prepareLayerLock.Unlock() - err = prepareLayer(&stdDriverInfo, path, layers) - if err != nil { - return hcserror.New(err, title+" - failed", "") - } - return nil -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/processimage.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/processimage.go deleted file mode 100644 index 884207c3e..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/processimage.go +++ /dev/null @@ -1,23 +0,0 @@ -package wclayer - -import "os" - -// ProcessBaseLayer post-processes a base layer that has had its files extracted. -// The files should have been extracted to \Files. -func ProcessBaseLayer(path string) error { - err := processBaseImage(path) - if err != nil { - return &os.PathError{Op: "ProcessBaseLayer", Path: path, Err: err} - } - return nil -} - -// ProcessUtilityVMImage post-processes a utility VM image that has had its files extracted. -// The files should have been extracted to \Files. -func ProcessUtilityVMImage(path string) error { - err := processUtilityImage(path) - if err != nil { - return &os.PathError{Op: "ProcessUtilityVMImage", Path: path, Err: err} - } - return nil -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/unpreparelayer.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/unpreparelayer.go deleted file mode 100644 index bccd45969..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/unpreparelayer.go +++ /dev/null @@ -1,30 +0,0 @@ -package wclayer - -import ( - "github.com/Microsoft/hcsshim/internal/hcserror" - "github.com/sirupsen/logrus" -) - -// UnprepareLayer disables the filesystem filter for the read-write layer with -// the given id. -func UnprepareLayer(path string) (err error) { - title := "hcsshim::UnprepareLayer" - fields := logrus.Fields{ - "path": path, - } - logrus.WithFields(fields).Debug(title) - defer func() { - if err != nil { - fields[logrus.ErrorKey] = err - logrus.WithFields(fields).Error(err) - } else { - logrus.WithFields(fields).Debug(title + " - succeeded") - } - }() - - err = unprepareLayer(&stdDriverInfo, path) - if err != nil { - return hcserror.New(err, title+" - failed", "") - } - return nil -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/wclayer.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/wclayer.go deleted file mode 100644 index 78f2aacd8..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/wclayer.go +++ /dev/null @@ -1,27 +0,0 @@ -package wclayer - -import "github.com/Microsoft/hcsshim/internal/guid" - -//go:generate go run ../../mksyscall_windows.go -output zsyscall_windows.go wclayer.go - -//sys activateLayer(info *driverInfo, id string) (hr error) = vmcompute.ActivateLayer? -//sys copyLayer(info *driverInfo, srcId string, dstId string, descriptors []WC_LAYER_DESCRIPTOR) (hr error) = vmcompute.CopyLayer? -//sys createLayer(info *driverInfo, id string, parent string) (hr error) = vmcompute.CreateLayer? -//sys createSandboxLayer(info *driverInfo, id string, parent uintptr, descriptors []WC_LAYER_DESCRIPTOR) (hr error) = vmcompute.CreateSandboxLayer? -//sys expandSandboxSize(info *driverInfo, id string, size uint64) (hr error) = vmcompute.ExpandSandboxSize? -//sys deactivateLayer(info *driverInfo, id string) (hr error) = vmcompute.DeactivateLayer? -//sys destroyLayer(info *driverInfo, id string) (hr error) = vmcompute.DestroyLayer? -//sys exportLayer(info *driverInfo, id string, path string, descriptors []WC_LAYER_DESCRIPTOR) (hr error) = vmcompute.ExportLayer? -//sys getLayerMountPath(info *driverInfo, id string, length *uintptr, buffer *uint16) (hr error) = vmcompute.GetLayerMountPath? -//sys getBaseImages(buffer **uint16) (hr error) = vmcompute.GetBaseImages? -//sys importLayer(info *driverInfo, id string, path string, descriptors []WC_LAYER_DESCRIPTOR) (hr error) = vmcompute.ImportLayer? -//sys layerExists(info *driverInfo, id string, exists *uint32) (hr error) = vmcompute.LayerExists? -//sys nameToGuid(name string, guid *_guid) (hr error) = vmcompute.NameToGuid? -//sys prepareLayer(info *driverInfo, id string, descriptors []WC_LAYER_DESCRIPTOR) (hr error) = vmcompute.PrepareLayer? -//sys unprepareLayer(info *driverInfo, id string) (hr error) = vmcompute.UnprepareLayer? -//sys processBaseImage(path string) (hr error) = vmcompute.ProcessBaseImage? -//sys processUtilityImage(path string) (hr error) = vmcompute.ProcessUtilityImage? - -//sys grantVmAccess(vmid string, filepath string) (hr error) = vmcompute.GrantVmAccess? - -type _guid = guid.GUID diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/zsyscall_windows.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/zsyscall_windows.go deleted file mode 100644 index d853ab259..000000000 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/zsyscall_windows.go +++ /dev/null @@ -1,510 +0,0 @@ -// Code generated mksyscall_windows.exe DO NOT EDIT - -package wclayer - -import ( - "syscall" - "unsafe" - - "golang.org/x/sys/windows" -) - -var _ unsafe.Pointer - -// Do the interface allocations only once for common -// Errno values. -const ( - errnoERROR_IO_PENDING = 997 -) - -var ( - errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING) -) - -// errnoErr returns common boxed Errno values, to prevent -// allocations at runtime. -func errnoErr(e syscall.Errno) error { - switch e { - case 0: - return nil - case errnoERROR_IO_PENDING: - return errERROR_IO_PENDING - } - // TODO: add more here, after collecting data on the common - // error values see on Windows. (perhaps when running - // all.bat?) - return e -} - -var ( - modvmcompute = windows.NewLazySystemDLL("vmcompute.dll") - - procActivateLayer = modvmcompute.NewProc("ActivateLayer") - procCopyLayer = modvmcompute.NewProc("CopyLayer") - procCreateLayer = modvmcompute.NewProc("CreateLayer") - procCreateSandboxLayer = modvmcompute.NewProc("CreateSandboxLayer") - procExpandSandboxSize = modvmcompute.NewProc("ExpandSandboxSize") - procDeactivateLayer = modvmcompute.NewProc("DeactivateLayer") - procDestroyLayer = modvmcompute.NewProc("DestroyLayer") - procExportLayer = modvmcompute.NewProc("ExportLayer") - procGetLayerMountPath = modvmcompute.NewProc("GetLayerMountPath") - procGetBaseImages = modvmcompute.NewProc("GetBaseImages") - procImportLayer = modvmcompute.NewProc("ImportLayer") - procLayerExists = modvmcompute.NewProc("LayerExists") - procNameToGuid = modvmcompute.NewProc("NameToGuid") - procPrepareLayer = modvmcompute.NewProc("PrepareLayer") - procUnprepareLayer = modvmcompute.NewProc("UnprepareLayer") - procProcessBaseImage = modvmcompute.NewProc("ProcessBaseImage") - procProcessUtilityImage = modvmcompute.NewProc("ProcessUtilityImage") - procGrantVmAccess = modvmcompute.NewProc("GrantVmAccess") -) - -func activateLayer(info *driverInfo, id string) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(id) - if hr != nil { - return - } - return _activateLayer(info, _p0) -} - -func _activateLayer(info *driverInfo, id *uint16) (hr error) { - if hr = procActivateLayer.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procActivateLayer.Addr(), 2, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), 0) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func copyLayer(info *driverInfo, srcId string, dstId string, descriptors []WC_LAYER_DESCRIPTOR) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(srcId) - if hr != nil { - return - } - var _p1 *uint16 - _p1, hr = syscall.UTF16PtrFromString(dstId) - if hr != nil { - return - } - return _copyLayer(info, _p0, _p1, descriptors) -} - -func _copyLayer(info *driverInfo, srcId *uint16, dstId *uint16, descriptors []WC_LAYER_DESCRIPTOR) (hr error) { - var _p2 *WC_LAYER_DESCRIPTOR - if len(descriptors) > 0 { - _p2 = &descriptors[0] - } - if hr = procCopyLayer.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall6(procCopyLayer.Addr(), 5, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(srcId)), uintptr(unsafe.Pointer(dstId)), uintptr(unsafe.Pointer(_p2)), uintptr(len(descriptors)), 0) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func createLayer(info *driverInfo, id string, parent string) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(id) - if hr != nil { - return - } - var _p1 *uint16 - _p1, hr = syscall.UTF16PtrFromString(parent) - if hr != nil { - return - } - return _createLayer(info, _p0, _p1) -} - -func _createLayer(info *driverInfo, id *uint16, parent *uint16) (hr error) { - if hr = procCreateLayer.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procCreateLayer.Addr(), 3, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(parent))) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func createSandboxLayer(info *driverInfo, id string, parent uintptr, descriptors []WC_LAYER_DESCRIPTOR) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(id) - if hr != nil { - return - } - return _createSandboxLayer(info, _p0, parent, descriptors) -} - -func _createSandboxLayer(info *driverInfo, id *uint16, parent uintptr, descriptors []WC_LAYER_DESCRIPTOR) (hr error) { - var _p1 *WC_LAYER_DESCRIPTOR - if len(descriptors) > 0 { - _p1 = &descriptors[0] - } - if hr = procCreateSandboxLayer.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall6(procCreateSandboxLayer.Addr(), 5, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), uintptr(parent), uintptr(unsafe.Pointer(_p1)), uintptr(len(descriptors)), 0) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func expandSandboxSize(info *driverInfo, id string, size uint64) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(id) - if hr != nil { - return - } - return _expandSandboxSize(info, _p0, size) -} - -func _expandSandboxSize(info *driverInfo, id *uint16, size uint64) (hr error) { - if hr = procExpandSandboxSize.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procExpandSandboxSize.Addr(), 3, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), uintptr(size)) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func deactivateLayer(info *driverInfo, id string) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(id) - if hr != nil { - return - } - return _deactivateLayer(info, _p0) -} - -func _deactivateLayer(info *driverInfo, id *uint16) (hr error) { - if hr = procDeactivateLayer.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procDeactivateLayer.Addr(), 2, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), 0) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func destroyLayer(info *driverInfo, id string) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(id) - if hr != nil { - return - } - return _destroyLayer(info, _p0) -} - -func _destroyLayer(info *driverInfo, id *uint16) (hr error) { - if hr = procDestroyLayer.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procDestroyLayer.Addr(), 2, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), 0) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func exportLayer(info *driverInfo, id string, path string, descriptors []WC_LAYER_DESCRIPTOR) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(id) - if hr != nil { - return - } - var _p1 *uint16 - _p1, hr = syscall.UTF16PtrFromString(path) - if hr != nil { - return - } - return _exportLayer(info, _p0, _p1, descriptors) -} - -func _exportLayer(info *driverInfo, id *uint16, path *uint16, descriptors []WC_LAYER_DESCRIPTOR) (hr error) { - var _p2 *WC_LAYER_DESCRIPTOR - if len(descriptors) > 0 { - _p2 = &descriptors[0] - } - if hr = procExportLayer.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall6(procExportLayer.Addr(), 5, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(path)), uintptr(unsafe.Pointer(_p2)), uintptr(len(descriptors)), 0) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func getLayerMountPath(info *driverInfo, id string, length *uintptr, buffer *uint16) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(id) - if hr != nil { - return - } - return _getLayerMountPath(info, _p0, length, buffer) -} - -func _getLayerMountPath(info *driverInfo, id *uint16, length *uintptr, buffer *uint16) (hr error) { - if hr = procGetLayerMountPath.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall6(procGetLayerMountPath.Addr(), 4, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(length)), uintptr(unsafe.Pointer(buffer)), 0, 0) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func getBaseImages(buffer **uint16) (hr error) { - if hr = procGetBaseImages.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procGetBaseImages.Addr(), 1, uintptr(unsafe.Pointer(buffer)), 0, 0) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func importLayer(info *driverInfo, id string, path string, descriptors []WC_LAYER_DESCRIPTOR) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(id) - if hr != nil { - return - } - var _p1 *uint16 - _p1, hr = syscall.UTF16PtrFromString(path) - if hr != nil { - return - } - return _importLayer(info, _p0, _p1, descriptors) -} - -func _importLayer(info *driverInfo, id *uint16, path *uint16, descriptors []WC_LAYER_DESCRIPTOR) (hr error) { - var _p2 *WC_LAYER_DESCRIPTOR - if len(descriptors) > 0 { - _p2 = &descriptors[0] - } - if hr = procImportLayer.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall6(procImportLayer.Addr(), 5, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(path)), uintptr(unsafe.Pointer(_p2)), uintptr(len(descriptors)), 0) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func layerExists(info *driverInfo, id string, exists *uint32) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(id) - if hr != nil { - return - } - return _layerExists(info, _p0, exists) -} - -func _layerExists(info *driverInfo, id *uint16, exists *uint32) (hr error) { - if hr = procLayerExists.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procLayerExists.Addr(), 3, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(exists))) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func nameToGuid(name string, guid *_guid) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(name) - if hr != nil { - return - } - return _nameToGuid(_p0, guid) -} - -func _nameToGuid(name *uint16, guid *_guid) (hr error) { - if hr = procNameToGuid.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procNameToGuid.Addr(), 2, uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(guid)), 0) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func prepareLayer(info *driverInfo, id string, descriptors []WC_LAYER_DESCRIPTOR) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(id) - if hr != nil { - return - } - return _prepareLayer(info, _p0, descriptors) -} - -func _prepareLayer(info *driverInfo, id *uint16, descriptors []WC_LAYER_DESCRIPTOR) (hr error) { - var _p1 *WC_LAYER_DESCRIPTOR - if len(descriptors) > 0 { - _p1 = &descriptors[0] - } - if hr = procPrepareLayer.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall6(procPrepareLayer.Addr(), 4, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(_p1)), uintptr(len(descriptors)), 0, 0) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func unprepareLayer(info *driverInfo, id string) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(id) - if hr != nil { - return - } - return _unprepareLayer(info, _p0) -} - -func _unprepareLayer(info *driverInfo, id *uint16) (hr error) { - if hr = procUnprepareLayer.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procUnprepareLayer.Addr(), 2, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(id)), 0) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func processBaseImage(path string) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(path) - if hr != nil { - return - } - return _processBaseImage(_p0) -} - -func _processBaseImage(path *uint16) (hr error) { - if hr = procProcessBaseImage.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procProcessBaseImage.Addr(), 1, uintptr(unsafe.Pointer(path)), 0, 0) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func processUtilityImage(path string) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(path) - if hr != nil { - return - } - return _processUtilityImage(_p0) -} - -func _processUtilityImage(path *uint16) (hr error) { - if hr = procProcessUtilityImage.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procProcessUtilityImage.Addr(), 1, uintptr(unsafe.Pointer(path)), 0, 0) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} - -func grantVmAccess(vmid string, filepath string) (hr error) { - var _p0 *uint16 - _p0, hr = syscall.UTF16PtrFromString(vmid) - if hr != nil { - return - } - var _p1 *uint16 - _p1, hr = syscall.UTF16PtrFromString(filepath) - if hr != nil { - return - } - return _grantVmAccess(_p0, _p1) -} - -func _grantVmAccess(vmid *uint16, filepath *uint16) (hr error) { - if hr = procGrantVmAccess.Find(); hr != nil { - return - } - r0, _, _ := syscall.Syscall(procGrantVmAccess.Addr(), 2, uintptr(unsafe.Pointer(vmid)), uintptr(unsafe.Pointer(filepath)), 0) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} diff --git a/vendor/github.com/Microsoft/hcsshim/layer.go b/vendor/github.com/Microsoft/hcsshim/layer.go deleted file mode 100644 index df0e63bbd..000000000 --- a/vendor/github.com/Microsoft/hcsshim/layer.go +++ /dev/null @@ -1,106 +0,0 @@ -package hcsshim - -import ( - "crypto/sha1" - "path/filepath" - - "github.com/Microsoft/hcsshim/internal/guid" - "github.com/Microsoft/hcsshim/internal/wclayer" -) - -func layerPath(info *DriverInfo, id string) string { - return filepath.Join(info.HomeDir, id) -} - -func ActivateLayer(info DriverInfo, id string) error { - return wclayer.ActivateLayer(layerPath(&info, id)) -} -func CreateLayer(info DriverInfo, id, parent string) error { - return wclayer.CreateLayer(layerPath(&info, id), parent) -} - -// New clients should use CreateScratchLayer instead. Kept in to preserve API compatibility. -func CreateSandboxLayer(info DriverInfo, layerId, parentId string, parentLayerPaths []string) error { - return wclayer.CreateScratchLayer(layerPath(&info, layerId), parentLayerPaths) -} -func CreateScratchLayer(info DriverInfo, layerId, parentId string, parentLayerPaths []string) error { - return wclayer.CreateScratchLayer(layerPath(&info, layerId), parentLayerPaths) -} -func DeactivateLayer(info DriverInfo, id string) error { - return wclayer.DeactivateLayer(layerPath(&info, id)) -} -func DestroyLayer(info DriverInfo, id string) error { - return wclayer.DestroyLayer(layerPath(&info, id)) -} - -// New clients should use ExpandScratchSize instead. Kept in to preserve API compatibility. -func ExpandSandboxSize(info DriverInfo, layerId string, size uint64) error { - return wclayer.ExpandScratchSize(layerPath(&info, layerId), size) -} -func ExpandScratchSize(info DriverInfo, layerId string, size uint64) error { - return wclayer.ExpandScratchSize(layerPath(&info, layerId), size) -} -func ExportLayer(info DriverInfo, layerId string, exportFolderPath string, parentLayerPaths []string) error { - return wclayer.ExportLayer(layerPath(&info, layerId), exportFolderPath, parentLayerPaths) -} -func GetLayerMountPath(info DriverInfo, id string) (string, error) { - return wclayer.GetLayerMountPath(layerPath(&info, id)) -} -func GetSharedBaseImages() (imageData string, err error) { - return wclayer.GetSharedBaseImages() -} -func ImportLayer(info DriverInfo, layerID string, importFolderPath string, parentLayerPaths []string) error { - return wclayer.ImportLayer(layerPath(&info, layerID), importFolderPath, parentLayerPaths) -} -func LayerExists(info DriverInfo, id string) (bool, error) { - return wclayer.LayerExists(layerPath(&info, id)) -} -func PrepareLayer(info DriverInfo, layerId string, parentLayerPaths []string) error { - return wclayer.PrepareLayer(layerPath(&info, layerId), parentLayerPaths) -} -func ProcessBaseLayer(path string) error { - return wclayer.ProcessBaseLayer(path) -} -func ProcessUtilityVMImage(path string) error { - return wclayer.ProcessUtilityVMImage(path) -} -func UnprepareLayer(info DriverInfo, layerId string) error { - return wclayer.UnprepareLayer(layerPath(&info, layerId)) -} - -type DriverInfo struct { - Flavour int - HomeDir string -} - -type GUID [16]byte - -func NameToGuid(name string) (id GUID, err error) { - g, err := wclayer.NameToGuid(name) - return GUID(g), err -} - -func NewGUID(source string) *GUID { - h := sha1.Sum([]byte(source)) - var g GUID - copy(g[0:], h[0:16]) - return &g -} - -func (g *GUID) ToString() string { - return (guid.GUID)(*g).String() -} - -type LayerReader = wclayer.LayerReader - -func NewLayerReader(info DriverInfo, layerID string, parentLayerPaths []string) (LayerReader, error) { - return wclayer.NewLayerReader(layerPath(&info, layerID), parentLayerPaths) -} - -type LayerWriter = wclayer.LayerWriter - -func NewLayerWriter(info DriverInfo, layerID string, parentLayerPaths []string) (LayerWriter, error) { - return wclayer.NewLayerWriter(layerPath(&info, layerID), parentLayerPaths) -} - -type WC_LAYER_DESCRIPTOR = wclayer.WC_LAYER_DESCRIPTOR diff --git a/vendor/github.com/Microsoft/hcsshim/process.go b/vendor/github.com/Microsoft/hcsshim/process.go deleted file mode 100644 index ca8acbb7c..000000000 --- a/vendor/github.com/Microsoft/hcsshim/process.go +++ /dev/null @@ -1,72 +0,0 @@ -package hcsshim - -import ( - "io" - "time" - - "github.com/Microsoft/hcsshim/internal/hcs" -) - -// ContainerError is an error encountered in HCS -type process struct { - p *hcs.Process -} - -// Pid returns the process ID of the process within the container. -func (process *process) Pid() int { - return process.p.Pid() -} - -// Kill signals the process to terminate but does not wait for it to finish terminating. -func (process *process) Kill() error { - return convertProcessError(process.p.Kill(), process) -} - -// Wait waits for the process to exit. -func (process *process) Wait() error { - return convertProcessError(process.p.Wait(), process) -} - -// WaitTimeout waits for the process to exit or the duration to elapse. It returns -// false if timeout occurs. -func (process *process) WaitTimeout(timeout time.Duration) error { - return convertProcessError(process.p.WaitTimeout(timeout), process) -} - -// ExitCode returns the exit code of the process. The process must have -// already terminated. -func (process *process) ExitCode() (int, error) { - code, err := process.p.ExitCode() - if err != nil { - err = convertProcessError(err, process) - } - return code, err -} - -// ResizeConsole resizes the console of the process. -func (process *process) ResizeConsole(width, height uint16) error { - return convertProcessError(process.p.ResizeConsole(width, height), process) -} - -// Stdio returns the stdin, stdout, and stderr pipes, respectively. Closing -// these pipes does not close the underlying pipes; it should be possible to -// call this multiple times to get multiple interfaces. -func (process *process) Stdio() (io.WriteCloser, io.ReadCloser, io.ReadCloser, error) { - stdin, stdout, stderr, err := process.p.Stdio() - if err != nil { - err = convertProcessError(err, process) - } - return stdin, stdout, stderr, err -} - -// CloseStdin closes the write side of the stdin pipe so that the process is -// notified on the read side that there is no more data in stdin. -func (process *process) CloseStdin() error { - return convertProcessError(process.p.CloseStdin(), process) -} - -// Close cleans up any state associated with the process but does not kill -// or wait on it. -func (process *process) Close() error { - return convertProcessError(process.p.Close(), process) -} diff --git a/vendor/github.com/Microsoft/hcsshim/vendor.conf b/vendor/github.com/Microsoft/hcsshim/vendor.conf deleted file mode 100644 index 6e0ed1566..000000000 --- a/vendor/github.com/Microsoft/hcsshim/vendor.conf +++ /dev/null @@ -1,21 +0,0 @@ -github.com/blang/semver v3.1.0 -github.com/containerd/console c12b1e7919c14469339a5d38f2f8ed9b64a9de23 -github.com/containerd/go-runc 5a6d9f37cfa36b15efba46dc7ea349fa9b7143c3 -github.com/hashicorp/errwrap 7554cd9344cec97297fa6649b055a8c98c2a1e55 -github.com/hashicorp/go-multierror ed905158d87462226a13fe39ddf685ea65f1c11f -github.com/konsorten/go-windows-terminal-sequences v1.0.1 -github.com/linuxkit/virtsock 8e79449dea0735c1c056d814934dd035734cc97c -github.com/Microsoft/go-winio 16cfc975803886a5e47c4257a24c8d8c52e178b2 -github.com/Microsoft/opengcs v0.3.9 -github.com/opencontainers/runtime-spec eba862dc2470385a233c7507392675cbeadf7353 -github.com/opencontainers/runtime-tools 1d69bd0f9c39677d0630e50664fbc3154ae61b88 -github.com/pkg/errors v0.8.1 -github.com/sirupsen/logrus v1.3.0 -github.com/syndtr/gocapability db04d3cc01c8b54962a58ec7e491717d06cfcc16 -github.com/urfave/cli 7bc6a0acffa589f415f88aca16cc1de5ffd66f9c -github.com/xeipuuv/gojsonpointer 4e3ac2762d5f479393488629ee9370b50873b3a6 -github.com/xeipuuv/gojsonreference bd5ef7bd5415a7ac448318e64f11a24cd21e594b -github.com/xeipuuv/gojsonschema 1d523034197ff1f222f6429836dd36a2457a1874 -golang.org/x/crypto ff983b9c42bc9fbf91556e191cc8efb585c16908 -golang.org/x/sync 37e7f081c4d4c64e13b10787722085407fe5d15f -golang.org/x/sys e5ecc2a6747ce8d4af18ed98b3de5ae30eb3a5bb \ No newline at end of file diff --git a/vendor/github.com/Microsoft/hcsshim/zsyscall_windows.go b/vendor/github.com/Microsoft/hcsshim/zsyscall_windows.go deleted file mode 100644 index 8bed84857..000000000 --- a/vendor/github.com/Microsoft/hcsshim/zsyscall_windows.go +++ /dev/null @@ -1,54 +0,0 @@ -// Code generated mksyscall_windows.exe DO NOT EDIT - -package hcsshim - -import ( - "syscall" - "unsafe" - - "golang.org/x/sys/windows" -) - -var _ unsafe.Pointer - -// Do the interface allocations only once for common -// Errno values. -const ( - errnoERROR_IO_PENDING = 997 -) - -var ( - errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING) -) - -// errnoErr returns common boxed Errno values, to prevent -// allocations at runtime. -func errnoErr(e syscall.Errno) error { - switch e { - case 0: - return nil - case errnoERROR_IO_PENDING: - return errERROR_IO_PENDING - } - // TODO: add more here, after collecting data on the common - // error values see on Windows. (perhaps when running - // all.bat?) - return e -} - -var ( - modiphlpapi = windows.NewLazySystemDLL("iphlpapi.dll") - - procSetCurrentThreadCompartmentId = modiphlpapi.NewProc("SetCurrentThreadCompartmentId") -) - -func SetCurrentThreadCompartmentId(compartmentId uint32) (hr error) { - r0, _, _ := syscall.Syscall(procSetCurrentThreadCompartmentId.Addr(), 1, uintptr(compartmentId), 0, 0) - if int32(r0) < 0 { - if r0&0x1fff0000 == 0x00070000 { - r0 &= 0xffff - } - hr = syscall.Errno(r0) - } - return -} diff --git a/vendor/github.com/containerd/containerd/archive/compression/compression.go b/vendor/github.com/containerd/containerd/archive/compression/compression.go index 2338de6b9..a883e4d58 100644 --- a/vendor/github.com/containerd/containerd/archive/compression/compression.go +++ b/vendor/github.com/containerd/containerd/archive/compression/compression.go @@ -29,6 +29,7 @@ import ( "sync" "github.com/containerd/containerd/log" + "github.com/klauspost/compress/zstd" ) type ( @@ -41,6 +42,8 @@ const ( Uncompressed Compression = iota // Gzip is gzip compression algorithm. Gzip + // Zstd is zstd compression algorithm. + Zstd ) const disablePigzEnv = "CONTAINERD_DISABLE_PIGZ" @@ -126,6 +129,7 @@ func (r *bufferedReader) Peek(n int) ([]byte, error) { func DetectCompression(source []byte) Compression { for compression, m := range map[Compression][]byte{ Gzip: {0x1F, 0x8B, 0x08}, + Zstd: {0x28, 0xb5, 0x2f, 0xfd}, } { if len(source) < len(m) { // Len too short @@ -174,6 +178,19 @@ func DecompressStream(archive io.Reader) (DecompressReadCloser, error) { return gzReader.Close() }, }, nil + case Zstd: + zstdReader, err := zstd.NewReader(buf) + if err != nil { + return nil, err + } + return &readCloserWrapper{ + Reader: zstdReader, + compression: compression, + closer: func() error { + zstdReader.Close() + return nil + }, + }, nil default: return nil, fmt.Errorf("unsupported compression format %s", (&compression).Extension()) @@ -187,6 +204,8 @@ func CompressStream(dest io.Writer, compression Compression) (io.WriteCloser, er return &writeCloserWrapper{dest, nil}, nil case Gzip: return gzip.NewWriter(dest), nil + case Zstd: + return zstd.NewWriter(dest) default: return nil, fmt.Errorf("unsupported compression format %s", (&compression).Extension()) } @@ -197,6 +216,8 @@ func (compression *Compression) Extension() string { switch *compression { case Gzip: return "gz" + case Zstd: + return "zst" } return "" } diff --git a/vendor/github.com/containerd/containerd/content/adaptor.go b/vendor/github.com/containerd/containerd/content/adaptor.go new file mode 100644 index 000000000..88bad2610 --- /dev/null +++ b/vendor/github.com/containerd/containerd/content/adaptor.go @@ -0,0 +1,52 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package content + +import ( + "strings" + + "github.com/containerd/containerd/filters" +) + +// AdaptInfo returns `filters.Adaptor` that handles `content.Info`. +func AdaptInfo(info Info) filters.Adaptor { + return filters.AdapterFunc(func(fieldpath []string) (string, bool) { + if len(fieldpath) == 0 { + return "", false + } + + switch fieldpath[0] { + case "digest": + return info.Digest.String(), true + case "size": + // TODO: support size based filtering + case "labels": + return checkMap(fieldpath[1:], info.Labels) + } + + return "", false + }) +} + +func checkMap(fieldpath []string, m map[string]string) (string, bool) { + if len(m) == 0 { + return "", false + } + + value, ok := m[strings.Join(fieldpath, ".")] + return value, ok +} diff --git a/vendor/github.com/containerd/containerd/content/content.go b/vendor/github.com/containerd/containerd/content/content.go index d8141a68b..ff17a8417 100644 --- a/vendor/github.com/containerd/containerd/content/content.go +++ b/vendor/github.com/containerd/containerd/content/content.go @@ -37,7 +37,7 @@ type Provider interface { // ReaderAt only requires desc.Digest to be set. // Other fields in the descriptor may be used internally for resolving // the location of the actual data. - ReaderAt(ctx context.Context, dec ocispec.Descriptor) (ReaderAt, error) + ReaderAt(ctx context.Context, desc ocispec.Descriptor) (ReaderAt, error) } // Ingester writes content diff --git a/vendor/github.com/containerd/containerd/content/helpers.go b/vendor/github.com/containerd/containerd/content/helpers.go index 4c4a35308..2af4159a4 100644 --- a/vendor/github.com/containerd/containerd/content/helpers.go +++ b/vendor/github.com/containerd/containerd/content/helpers.go @@ -25,11 +25,17 @@ import ( "time" "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/log" "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) +// maxResets is the no.of times the Copy() method can tolerate a reset of the body +const maxResets = 5 + +var ErrReset = errors.New("writer has been reset") + var bufPool = sync.Pool{ New: func() interface{} { buffer := make([]byte, 1<<20) @@ -80,7 +86,7 @@ func WriteBlob(ctx context.Context, cs Ingester, ref string, r io.Reader, desc o return errors.Wrap(err, "failed to open writer") } - return nil // all ready present + return nil // already present } defer cw.Close() @@ -131,30 +137,63 @@ func OpenWriter(ctx context.Context, cs Ingester, opts ...WriterOpt) (Writer, er // the size or digest is unknown, these values may be empty. // // Copy is buffered, so no need to wrap reader in buffered io. -func Copy(ctx context.Context, cw Writer, r io.Reader, size int64, expected digest.Digest, opts ...Opt) error { +func Copy(ctx context.Context, cw Writer, or io.Reader, size int64, expected digest.Digest, opts ...Opt) error { ws, err := cw.Status() if err != nil { return errors.Wrap(err, "failed to get status") } - + r := or if ws.Offset > 0 { - r, err = seekReader(r, ws.Offset, size) + r, err = seekReader(or, ws.Offset, size) if err != nil { return errors.Wrapf(err, "unable to resume write to %v", ws.Ref) } } - if _, err := copyWithBuffer(cw, r); err != nil { - return errors.Wrap(err, "failed to copy") - } - - if err := cw.Commit(ctx, size, expected, opts...); err != nil { - if !errdefs.IsAlreadyExists(err) { - return errors.Wrapf(err, "failed commit on ref %q", ws.Ref) + for i := 0; i < maxResets; i++ { + if i >= 1 { + log.G(ctx).WithField("digest", expected).Debugf("retrying copy due to reset") } + copied, err := copyWithBuffer(cw, r) + if errors.Is(err, ErrReset) { + ws, err := cw.Status() + if err != nil { + return errors.Wrap(err, "failed to get status") + } + r, err = seekReader(or, ws.Offset, size) + if err != nil { + return errors.Wrapf(err, "unable to resume write to %v", ws.Ref) + } + continue + } + if err != nil { + return errors.Wrap(err, "failed to copy") + } + if size != 0 && copied < size-ws.Offset { + // Short writes would return its own error, this indicates a read failure + errors.Wrapf(io.ErrUnexpectedEOF, "failed to read expected number of bytes") + } + if err := cw.Commit(ctx, size, expected, opts...); err != nil { + if errors.Is(err, ErrReset) { + ws, err := cw.Status() + if err != nil { + return errors.Wrap(err, "failed to get status: %w") + } + r, err = seekReader(or, ws.Offset, size) + if err != nil { + return errors.Wrapf(err, "unable to resume write to %v", ws.Ref) + } + continue + } + if !errdefs.IsAlreadyExists(err) { + return errors.Wrapf(err, "failed commit on ref %q", ws.Ref) + } + } + return nil } - return nil + log.G(ctx).WithField("digest", expected).Errorf("failed to copy after %d retries", maxResets) + return errors.Errorf("failed to copy after %d retries", maxResets) } // CopyReaderAt copies to a writer from a given reader at for the given @@ -165,8 +204,15 @@ func CopyReaderAt(cw Writer, ra ReaderAt, n int64) error { return err } - _, err = copyWithBuffer(cw, io.NewSectionReader(ra, ws.Offset, n)) - return err + copied, err := copyWithBuffer(cw, io.NewSectionReader(ra, ws.Offset, n)) + if err != nil { + return errors.Wrap(err, "failed to copy") + } + if copied < n { + // Short writes would return its own error, this indicates a read failure + return errors.Wrap(io.ErrUnexpectedEOF, "failed to read expected number of bytes") + } + return nil } // CopyReader copies to a writer from a given reader, returning diff --git a/vendor/github.com/containerd/containerd/content/local/locks.go b/vendor/github.com/containerd/containerd/content/local/locks.go index bc3bd18e0..d1d2d564d 100644 --- a/vendor/github.com/containerd/containerd/content/local/locks.go +++ b/vendor/github.com/containerd/containerd/content/local/locks.go @@ -18,6 +18,7 @@ package local import ( "sync" + "time" "github.com/containerd/containerd/errdefs" "github.com/pkg/errors" @@ -25,9 +26,13 @@ import ( // Handles locking references +type lock struct { + since time.Time +} + var ( // locks lets us lock in process - locks = map[string]struct{}{} + locks = make(map[string]*lock) locksMu sync.Mutex ) @@ -35,11 +40,11 @@ func tryLock(ref string) error { locksMu.Lock() defer locksMu.Unlock() - if _, ok := locks[ref]; ok { - return errors.Wrapf(errdefs.ErrUnavailable, "ref %s locked", ref) + if v, ok := locks[ref]; ok { + return errors.Wrapf(errdefs.ErrUnavailable, "ref %s locked since %s", ref, v.since) } - locks[ref] = struct{}{} + locks[ref] = &lock{time.Now()} return nil } diff --git a/vendor/github.com/containerd/containerd/content/local/readerat.go b/vendor/github.com/containerd/containerd/content/local/readerat.go index 42b99dc42..5d3ae0390 100644 --- a/vendor/github.com/containerd/containerd/content/local/readerat.go +++ b/vendor/github.com/containerd/containerd/content/local/readerat.go @@ -18,6 +18,11 @@ package local import ( "os" + + "github.com/pkg/errors" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/errdefs" ) // readerat implements io.ReaderAt in a completely stateless manner by opening @@ -27,6 +32,29 @@ type sizeReaderAt struct { fp *os.File } +// OpenReader creates ReaderAt from a file +func OpenReader(p string) (content.ReaderAt, error) { + fi, err := os.Stat(p) + if err != nil { + if !os.IsNotExist(err) { + return nil, err + } + + return nil, errors.Wrap(errdefs.ErrNotFound, "blob not found") + } + + fp, err := os.Open(p) + if err != nil { + if !os.IsNotExist(err) { + return nil, err + } + + return nil, errors.Wrap(errdefs.ErrNotFound, "blob not found") + } + + return sizeReaderAt{size: fi.Size(), fp: fp}, nil +} + func (ra sizeReaderAt) ReadAt(p []byte, offset int64) (int, error) { return ra.fp.ReadAt(p, offset) } diff --git a/vendor/github.com/containerd/containerd/content/local/store.go b/vendor/github.com/containerd/containerd/content/local/store.go index 15f7b25b0..78fd9cc9c 100644 --- a/vendor/github.com/containerd/containerd/content/local/store.go +++ b/vendor/github.com/containerd/containerd/content/local/store.go @@ -131,25 +131,13 @@ func (s *store) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content. if err != nil { return nil, errors.Wrapf(err, "calculating blob path for ReaderAt") } - fi, err := os.Stat(p) - if err != nil { - if !os.IsNotExist(err) { - return nil, err - } - return nil, errors.Wrapf(errdefs.ErrNotFound, "blob %s expected at %s", desc.Digest, p) + reader, err := OpenReader(p) + if err != nil { + return nil, errors.Wrapf(err, "blob %s expected at %s", desc.Digest, p) } - fp, err := os.Open(p) - if err != nil { - if !os.IsNotExist(err) { - return nil, err - } - - return nil, errors.Wrapf(errdefs.ErrNotFound, "blob %s expected at %s", desc.Digest, p) - } - - return sizeReaderAt{size: fi.Size(), fp: fp}, nil + return reader, nil } // Delete removes a blob by its digest. @@ -240,9 +228,14 @@ func (s *store) Update(ctx context.Context, info content.Info, fieldpaths ...str return info, nil } -func (s *store) Walk(ctx context.Context, fn content.WalkFunc, filters ...string) error { - // TODO: Support filters +func (s *store) Walk(ctx context.Context, fn content.WalkFunc, fs ...string) error { root := filepath.Join(s.root, "blobs") + + filter, err := filters.ParseAll(fs...) + if err != nil { + return err + } + var alg digest.Algorithm return filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { if err != nil { @@ -286,7 +279,12 @@ func (s *store) Walk(ctx context.Context, fn content.WalkFunc, filters ...string return err } } - return fn(s.info(dgst, fi, labels)) + + info := s.info(dgst, fi, labels) + if !filter.Match(content.AdaptInfo(info)) { + return nil + } + return fn(info) }) } @@ -467,7 +465,6 @@ func (s *store) Writer(ctx context.Context, opts ...content.WriterOpt) (content. } var lockErr error for count := uint64(0); count < 10; count++ { - time.Sleep(time.Millisecond * time.Duration(rand.Intn(1<` -// (see https://msdn.microsoft.com/en-us/library/windows/desktop/aa365150) -func GetLocalListener(path string, uid, gid int) (net.Listener, error) { - return winio.ListenPipe(path, nil) +func getATime(fi os.FileInfo) time.Time { + if st, ok := fi.Sys().(*syscall.Stat_t); ok { + return time.Unix(int64(st.Atimespec.Sec), int64(st.Atimespec.Nsec)) //nolint: unconvert // int64 conversions ensure the line compiles for 32-bit systems as well. + } + + return fi.ModTime() } diff --git a/vendor/github.com/containerd/containerd/sys/fds.go b/vendor/github.com/containerd/containerd/content/local/store_openbsd.go similarity index 64% rename from vendor/github.com/containerd/containerd/sys/fds.go rename to vendor/github.com/containerd/containerd/content/local/store_openbsd.go index db3cf702f..05d1a1a98 100644 --- a/vendor/github.com/containerd/containerd/sys/fds.go +++ b/vendor/github.com/containerd/containerd/content/local/store_openbsd.go @@ -1,4 +1,5 @@ -// +build !windows,!darwin +//go:build openbsd +// +build openbsd /* Copyright The containerd Authors. @@ -16,19 +17,18 @@ limitations under the License. */ -package sys +package local import ( - "io/ioutil" - "path/filepath" - "strconv" + "os" + "syscall" + "time" ) -// GetOpenFds returns the number of open fds for the process provided by pid -func GetOpenFds(pid int) (int, error) { - dirs, err := ioutil.ReadDir(filepath.Join("/proc", strconv.Itoa(pid), "fd")) - if err != nil { - return -1, err +func getATime(fi os.FileInfo) time.Time { + if st, ok := fi.Sys().(*syscall.Stat_t); ok { + return time.Unix(int64(st.Atim.Sec), int64(st.Atim.Nsec)) //nolint: unconvert // int64 conversions ensure the line compiles for 32-bit systems as well. } - return len(dirs), nil + + return fi.ModTime() } diff --git a/vendor/github.com/containerd/containerd/content/local/store_unix.go b/vendor/github.com/containerd/containerd/content/local/store_unix.go index f5f34fd0c..d60d753cb 100644 --- a/vendor/github.com/containerd/containerd/content/local/store_unix.go +++ b/vendor/github.com/containerd/containerd/content/local/store_unix.go @@ -1,4 +1,5 @@ -// +build linux solaris darwin freebsd +//go:build linux || solaris +// +build linux solaris /* Copyright The containerd Authors. @@ -22,13 +23,11 @@ import ( "os" "syscall" "time" - - "github.com/containerd/containerd/sys" ) func getATime(fi os.FileInfo) time.Time { if st, ok := fi.Sys().(*syscall.Stat_t); ok { - return sys.StatATimeAsTime(st) + return time.Unix(int64(st.Atim.Sec), int64(st.Atim.Nsec)) //nolint: unconvert // int64 conversions ensure the line compiles for 32-bit systems as well. } return fi.ModTime() diff --git a/vendor/github.com/containerd/containerd/filters/filter.go b/vendor/github.com/containerd/containerd/filters/filter.go index cf09d8d9e..e13f2625c 100644 --- a/vendor/github.com/containerd/containerd/filters/filter.go +++ b/vendor/github.com/containerd/containerd/filters/filter.go @@ -65,7 +65,6 @@ // ``` // name==foo,labels.bar // ``` -// package filters import ( diff --git a/vendor/github.com/containerd/containerd/filters/parser.go b/vendor/github.com/containerd/containerd/filters/parser.go index 0825d668c..eb48936d5 100644 --- a/vendor/github.com/containerd/containerd/filters/parser.go +++ b/vendor/github.com/containerd/containerd/filters/parser.go @@ -46,7 +46,6 @@ field := quoted | [A-Za-z] [A-Za-z0-9_]+ operator := "==" | "!=" | "~=" value := quoted | [^\s,]+ quoted := - */ func Parse(s string) (Filter, error) { // special case empty to match all diff --git a/vendor/github.com/containerd/containerd/filters/quote.go b/vendor/github.com/containerd/containerd/filters/quote.go index 2d64e23a3..bbee5e449 100644 --- a/vendor/github.com/containerd/containerd/filters/quote.go +++ b/vendor/github.com/containerd/containerd/filters/quote.go @@ -32,10 +32,10 @@ var errQuoteSyntax = errors.New("quote syntax error") // or character literal represented by the string s. // It returns four values: // -// 1) value, the decoded Unicode code point or byte value; -// 2) multibyte, a boolean indicating whether the decoded character requires a multibyte UTF-8 representation; -// 3) tail, the remainder of the string after the character; and -// 4) an error that will be nil if the character is syntactically valid. +// 1. value, the decoded Unicode code point or byte value; +// 2. multibyte, a boolean indicating whether the decoded character requires a multibyte UTF-8 representation; +// 3. tail, the remainder of the string after the character; and +// 4. an error that will be nil if the character is syntactically valid. // // The second argument, quote, specifies the type of literal being parsed // and therefore which escaped quote character is permitted. diff --git a/vendor/github.com/containerd/containerd/images/diffid.go b/vendor/github.com/containerd/containerd/images/diffid.go new file mode 100644 index 000000000..56193cc28 --- /dev/null +++ b/vendor/github.com/containerd/containerd/images/diffid.go @@ -0,0 +1,81 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package images + +import ( + "context" + "io" + + "github.com/containerd/containerd/archive/compression" + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/labels" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/sirupsen/logrus" +) + +// GetDiffID gets the diff ID of the layer blob descriptor. +func GetDiffID(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (digest.Digest, error) { + switch desc.MediaType { + case + // If the layer is already uncompressed, we can just return its digest + MediaTypeDockerSchema2Layer, + ocispec.MediaTypeImageLayer, + MediaTypeDockerSchema2LayerForeign, + ocispec.MediaTypeImageLayerNonDistributable: + return desc.Digest, nil + } + info, err := cs.Info(ctx, desc.Digest) + if err != nil { + return "", err + } + v, ok := info.Labels[labels.LabelUncompressed] + if ok { + // Fast path: if the image is already unpacked, we can use the label value + return digest.Parse(v) + } + // if the image is not unpacked, we may not have the label + ra, err := cs.ReaderAt(ctx, desc) + if err != nil { + return "", err + } + defer ra.Close() + r := content.NewReader(ra) + uR, err := compression.DecompressStream(r) + if err != nil { + return "", err + } + defer uR.Close() + digester := digest.Canonical.Digester() + hashW := digester.Hash() + if _, err := io.Copy(hashW, uR); err != nil { + return "", err + } + if err := ra.Close(); err != nil { + return "", err + } + digest := digester.Digest() + // memorize the computed value + if info.Labels == nil { + info.Labels = make(map[string]string) + } + info.Labels[labels.LabelUncompressed] = digest.String() + if _, err := cs.Update(ctx, info, "labels"); err != nil { + logrus.WithError(err).Warnf("failed to set %s label for %s", labels.LabelUncompressed, desc.Digest) + } + return digest, nil +} diff --git a/vendor/github.com/containerd/containerd/images/image.go b/vendor/github.com/containerd/containerd/images/image.go index 2e42ca09a..2e5cd61c9 100644 --- a/vendor/github.com/containerd/containerd/images/image.go +++ b/vendor/github.com/containerd/containerd/images/image.go @@ -298,7 +298,7 @@ func Platforms(ctx context.Context, provider content.Provider, image ocispec.Des // If available is true, the caller can assume that required represents the // complete set of content required for the image. // -// missing will have the components that are part of required but not avaiiable +// missing will have the components that are part of required but not available // in the provider. // // If there is a problem resolving content, an error will be returned. diff --git a/vendor/github.com/containerd/containerd/images/mediatypes.go b/vendor/github.com/containerd/containerd/images/mediatypes.go index c51897d25..785d71291 100644 --- a/vendor/github.com/containerd/containerd/images/mediatypes.go +++ b/vendor/github.com/containerd/containerd/images/mediatypes.go @@ -49,6 +49,9 @@ const ( MediaTypeContainerd1CheckpointRuntimeOptions = "application/vnd.containerd.container.checkpoint.runtime.options+proto" // Legacy Docker schema1 manifest MediaTypeDockerSchema1Manifest = "application/vnd.docker.distribution.manifest.v1+prettyjws" + // Encypted media types + MediaTypeImageLayerEncrypted = ocispec.MediaTypeImageLayer + "+encrypted" + MediaTypeImageLayerGzipEncrypted = ocispec.MediaTypeImageLayerGzip + "+encrypted" ) // DiffCompression returns the compression as defined by the layer diff media @@ -78,6 +81,8 @@ func DiffCompression(ctx context.Context, mediaType string) (string, error) { switch ext[len(ext)-1] { case "gzip": return "gzip", nil + case "zstd": + return "zstd", nil } } return "", nil @@ -100,7 +105,13 @@ func parseMediaTypes(mt string) (string, []string) { return s[0], ext } -// IsLayerTypes returns true if the media type is a layer +// IsNonDistributable returns true if the media type is non-distributable. +func IsNonDistributable(mt string) bool { + return strings.HasPrefix(mt, "application/vnd.oci.image.layer.nondistributable.") || + strings.HasPrefix(mt, "application/vnd.docker.image.rootfs.foreign.") +} + +// IsLayerType returns true if the media type is a layer func IsLayerType(mt string) bool { if strings.HasPrefix(mt, "application/vnd.oci.image.layer.") { return true @@ -116,7 +127,45 @@ func IsLayerType(mt string) bool { return false } -// IsKnownConfig returns true if the media type is a known config type +// IsDockerType returns true if the media type has "application/vnd.docker." prefix +func IsDockerType(mt string) bool { + return strings.HasPrefix(mt, "application/vnd.docker.") +} + +// IsManifestType returns true if the media type is an OCI-compatible manifest. +// No support for schema1 manifest. +func IsManifestType(mt string) bool { + switch mt { + case MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest: + return true + default: + return false + } +} + +// IsIndexType returns true if the media type is an OCI-compatible index. +func IsIndexType(mt string) bool { + switch mt { + case ocispec.MediaTypeImageIndex, MediaTypeDockerSchema2ManifestList: + return true + default: + return false + } +} + +// IsConfigType returns true if the media type is an OCI-compatible image config. +// No support for containerd checkpoint configs. +func IsConfigType(mt string) bool { + switch mt { + case MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig: + return true + default: + return false + } +} + +// IsKnownConfig returns true if the media type is a known config type, +// including containerd checkpoint configs func IsKnownConfig(mt string) bool { switch mt { case MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig, diff --git a/vendor/github.com/containerd/containerd/sys/userns_unsupported.go b/vendor/github.com/containerd/containerd/labels/labels.go similarity index 76% rename from vendor/github.com/containerd/containerd/sys/userns_unsupported.go rename to vendor/github.com/containerd/containerd/labels/labels.go index 549b50200..d76ff2cf9 100644 --- a/vendor/github.com/containerd/containerd/sys/userns_unsupported.go +++ b/vendor/github.com/containerd/containerd/labels/labels.go @@ -1,5 +1,3 @@ -// +build !linux - /* Copyright The containerd Authors. @@ -16,10 +14,8 @@ limitations under the License. */ -package sys +package labels -// RunningInUserNS is a stub for non-Linux systems -// Always returns false -func RunningInUserNS() bool { - return false -} +// LabelUncompressed is added to compressed layer contents. +// The value is digest of the uncompressed content. +const LabelUncompressed = "containerd.io/uncompressed" diff --git a/vendor/github.com/containerd/containerd/log/context.go b/vendor/github.com/containerd/containerd/log/context.go index 21599c4fd..37b6a7d1c 100644 --- a/vendor/github.com/containerd/containerd/log/context.go +++ b/vendor/github.com/containerd/containerd/log/context.go @@ -37,9 +37,17 @@ type ( loggerKey struct{} ) -// RFC3339NanoFixed is time.RFC3339Nano with nanoseconds padded using zeros to -// ensure the formatted time is always the same number of characters. -const RFC3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00" +const ( + // RFC3339NanoFixed is time.RFC3339Nano with nanoseconds padded using zeros to + // ensure the formatted time is always the same number of characters. + RFC3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00" + + // TextFormat represents the text logging format + TextFormat = "text" + + // JSONFormat represents the JSON logging format + JSONFormat = "json" +) // WithLogger returns a new context with the provided logger. Use in // combination with logger.WithField(s) for great effect. diff --git a/vendor/github.com/containerd/containerd/platforms/compare.go b/vendor/github.com/containerd/containerd/platforms/compare.go index 3ad22a10d..c7657e186 100644 --- a/vendor/github.com/containerd/containerd/platforms/compare.go +++ b/vendor/github.com/containerd/containerd/platforms/compare.go @@ -16,7 +16,12 @@ package platforms -import specs "github.com/opencontainers/image-spec/specs-go/v1" +import ( + "strconv" + "strings" + + specs "github.com/opencontainers/image-spec/specs-go/v1" +) // MatchComparer is able to match and compare platforms to // filter and sort platforms. @@ -26,103 +31,70 @@ type MatchComparer interface { Less(specs.Platform, specs.Platform) bool } +// platformVector returns an (ordered) vector of appropriate specs.Platform +// objects to try matching for the given platform object (see platforms.Only). +func platformVector(platform specs.Platform) []specs.Platform { + vector := []specs.Platform{platform} + + switch platform.Architecture { + case "amd64": + vector = append(vector, specs.Platform{ + Architecture: "386", + OS: platform.OS, + OSVersion: platform.OSVersion, + OSFeatures: platform.OSFeatures, + Variant: platform.Variant, + }) + case "arm": + if armVersion, err := strconv.Atoi(strings.TrimPrefix(platform.Variant, "v")); err == nil && armVersion > 5 { + for armVersion--; armVersion >= 5; armVersion-- { + vector = append(vector, specs.Platform{ + Architecture: platform.Architecture, + OS: platform.OS, + OSVersion: platform.OSVersion, + OSFeatures: platform.OSFeatures, + Variant: "v" + strconv.Itoa(armVersion), + }) + } + } + case "arm64": + variant := platform.Variant + if variant == "" { + variant = "v8" + } + vector = append(vector, platformVector(specs.Platform{ + Architecture: "arm", + OS: platform.OS, + OSVersion: platform.OSVersion, + OSFeatures: platform.OSFeatures, + Variant: variant, + })...) + } + + return vector +} + // Only returns a match comparer for a single platform // using default resolution logic for the platform. // -// For ARMv8, will also match ARMv7, ARMv6 and ARMv5 (for 32bit runtimes) -// For ARMv7, will also match ARMv6 and ARMv5 -// For ARMv6, will also match ARMv5 +// For arm/v8, will also match arm/v7, arm/v6 and arm/v5 +// For arm/v7, will also match arm/v6 and arm/v5 +// For arm/v6, will also match arm/v5 +// For amd64, will also match 386 func Only(platform specs.Platform) MatchComparer { - platform = Normalize(platform) - if platform.Architecture == "arm" { - if platform.Variant == "v8" { - return orderedPlatformComparer{ - matchers: []Matcher{ - &matcher{ - Platform: platform, - }, - &matcher{ - Platform: specs.Platform{ - Architecture: platform.Architecture, - OS: platform.OS, - OSVersion: platform.OSVersion, - OSFeatures: platform.OSFeatures, - Variant: "v7", - }, - }, - &matcher{ - Platform: specs.Platform{ - Architecture: platform.Architecture, - OS: platform.OS, - OSVersion: platform.OSVersion, - OSFeatures: platform.OSFeatures, - Variant: "v6", - }, - }, - &matcher{ - Platform: specs.Platform{ - Architecture: platform.Architecture, - OS: platform.OS, - OSVersion: platform.OSVersion, - OSFeatures: platform.OSFeatures, - Variant: "v5", - }, - }, - }, - } - } - if platform.Variant == "v7" { - return orderedPlatformComparer{ - matchers: []Matcher{ - &matcher{ - Platform: platform, - }, - &matcher{ - Platform: specs.Platform{ - Architecture: platform.Architecture, - OS: platform.OS, - OSVersion: platform.OSVersion, - OSFeatures: platform.OSFeatures, - Variant: "v6", - }, - }, - &matcher{ - Platform: specs.Platform{ - Architecture: platform.Architecture, - OS: platform.OS, - OSVersion: platform.OSVersion, - OSFeatures: platform.OSFeatures, - Variant: "v5", - }, - }, - }, - } - } - if platform.Variant == "v6" { - return orderedPlatformComparer{ - matchers: []Matcher{ - &matcher{ - Platform: platform, - }, - &matcher{ - Platform: specs.Platform{ - Architecture: platform.Architecture, - OS: platform.OS, - OSVersion: platform.OSVersion, - OSFeatures: platform.OSFeatures, - Variant: "v5", - }, - }, - }, - } - } - } + return Ordered(platformVector(Normalize(platform))...) +} - return singlePlatformComparer{ - Matcher: &matcher{ - Platform: platform, - }, - } +// OnlyStrict returns a match comparer for a single platform. +// +// Unlike Only, OnlyStrict does not match sub platforms. +// So, "arm/vN" will not match "arm/vM" where M < N, +// and "amd64" will not also match "386". +// +// OnlyStrict matches non-canonical forms. +// So, "arm64" matches "arm/64/v8". +func OnlyStrict(platform specs.Platform) MatchComparer { + return Ordered(Normalize(platform)) } // Ordered returns a platform MatchComparer which matches any of the platforms @@ -153,14 +125,6 @@ func Any(platforms ...specs.Platform) MatchComparer { // with preference for ordering. var All MatchComparer = allPlatformComparer{} -type singlePlatformComparer struct { - Matcher -} - -func (c singlePlatformComparer) Less(p1, p2 specs.Platform) bool { - return c.Match(p1) && !c.Match(p2) -} - type orderedPlatformComparer struct { matchers []Matcher } diff --git a/vendor/github.com/containerd/containerd/platforms/cpuinfo.go b/vendor/github.com/containerd/containerd/platforms/cpuinfo.go index db65a726b..4a7177e31 100644 --- a/vendor/github.com/containerd/containerd/platforms/cpuinfo.go +++ b/vendor/github.com/containerd/containerd/platforms/cpuinfo.go @@ -21,6 +21,7 @@ import ( "os" "runtime" "strings" + "sync" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/log" @@ -28,14 +29,18 @@ import ( ) // Present the ARM instruction set architecture, eg: v7, v8 -var cpuVariant string +// Don't use this value directly; call cpuVariant() instead. +var cpuVariantValue string -func init() { - if isArmArch(runtime.GOARCH) { - cpuVariant = getCPUVariant() - } else { - cpuVariant = "" - } +var cpuVariantOnce sync.Once + +func cpuVariant() string { + cpuVariantOnce.Do(func() { + if isArmArch(runtime.GOARCH) { + cpuVariantValue = getCPUVariant() + } + }) + return cpuVariantValue } // For Linux, the kernel has already detected the ABI, ISA and Features. @@ -96,14 +101,18 @@ func getCPUVariant() string { return "" } + // handle edge case for Raspberry Pi ARMv6 devices (which due to a kernel quirk, report "CPU architecture: 7") + // https://www.raspberrypi.org/forums/viewtopic.php?t=12614 + if runtime.GOARCH == "arm" && variant == "7" { + model, err := getCPUInfo("model name") + if err == nil && strings.HasPrefix(strings.ToLower(model), "armv6-compatible") { + variant = "6" + } + } + switch strings.ToLower(variant) { case "8", "aarch64": - // special case: if running a 32-bit userspace on aarch64, the variant should be "v7" - if runtime.GOARCH == "arm" { - variant = "v7" - } else { - variant = "v8" - } + variant = "v8" case "7", "7m", "?(12)", "?(13)", "?(14)", "?(15)", "?(16)", "?(17)": variant = "v7" case "6", "6tej": diff --git a/vendor/github.com/containerd/containerd/platforms/defaults.go b/vendor/github.com/containerd/containerd/platforms/defaults.go index a14d80e58..cb77fbc9f 100644 --- a/vendor/github.com/containerd/containerd/platforms/defaults.go +++ b/vendor/github.com/containerd/containerd/platforms/defaults.go @@ -33,6 +33,11 @@ func DefaultSpec() specs.Platform { OS: runtime.GOOS, Architecture: runtime.GOARCH, // The Variant field will be empty if arch != ARM. - Variant: cpuVariant, + Variant: cpuVariant(), } } + +// DefaultStrict returns strict form of Default. +func DefaultStrict() MatchComparer { + return OnlyStrict(DefaultSpec()) +} diff --git a/vendor/github.com/containerd/containerd/platforms/defaults_unix.go b/vendor/github.com/containerd/containerd/platforms/defaults_unix.go index e8a7d5ffa..359063efd 100644 --- a/vendor/github.com/containerd/containerd/platforms/defaults_unix.go +++ b/vendor/github.com/containerd/containerd/platforms/defaults_unix.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows /* diff --git a/vendor/github.com/containerd/containerd/platforms/defaults_windows.go b/vendor/github.com/containerd/containerd/platforms/defaults_windows.go index 0defbd36c..413d19af5 100644 --- a/vendor/github.com/containerd/containerd/platforms/defaults_windows.go +++ b/vendor/github.com/containerd/containerd/platforms/defaults_windows.go @@ -1,3 +1,4 @@ +//go:build windows // +build windows /* @@ -19,13 +20,63 @@ package platforms import ( + "fmt" + "runtime" + "strconv" + "strings" + + imagespec "github.com/opencontainers/image-spec/specs-go/v1" specs "github.com/opencontainers/image-spec/specs-go/v1" + "golang.org/x/sys/windows" ) -// Default returns the default matcher for the platform. -func Default() MatchComparer { - return Ordered(DefaultSpec(), specs.Platform{ - OS: "linux", - Architecture: "amd64", - }) +type matchComparer struct { + defaults Matcher + osVersionPrefix string +} + +// Match matches platform with the same windows major, minor +// and build version. +func (m matchComparer) Match(p imagespec.Platform) bool { + if m.defaults.Match(p) { + // TODO(windows): Figure out whether OSVersion is deprecated. + return strings.HasPrefix(p.OSVersion, m.osVersionPrefix) + } + return false +} + +// Less sorts matched platforms in front of other platforms. +// For matched platforms, it puts platforms with larger revision +// number in front. +func (m matchComparer) Less(p1, p2 imagespec.Platform) bool { + m1, m2 := m.Match(p1), m.Match(p2) + if m1 && m2 { + r1, r2 := revision(p1.OSVersion), revision(p2.OSVersion) + return r1 > r2 + } + return m1 && !m2 +} + +func revision(v string) int { + parts := strings.Split(v, ".") + if len(parts) < 4 { + return 0 + } + r, err := strconv.Atoi(parts[3]) + if err != nil { + return 0 + } + return r +} + +// Default returns the current platform's default platform specification. +func Default() MatchComparer { + major, minor, build := windows.RtlGetNtVersionNumbers() + return matchComparer{ + defaults: Ordered(DefaultSpec(), specs.Platform{ + OS: "linux", + Architecture: runtime.GOARCH, + }), + osVersionPrefix: fmt.Sprintf("%d.%d.%d", major, minor, build), + } } diff --git a/vendor/github.com/containerd/containerd/platforms/platforms.go b/vendor/github.com/containerd/containerd/platforms/platforms.go index 77d3f184e..f08f6045b 100644 --- a/vendor/github.com/containerd/containerd/platforms/platforms.go +++ b/vendor/github.com/containerd/containerd/platforms/platforms.go @@ -27,40 +27,40 @@ // The vast majority of use cases should simply use the match function with // user input. The first step is to parse a specifier into a matcher: // -// m, err := Parse("linux") -// if err != nil { ... } +// m, err := Parse("linux") +// if err != nil { ... } // // Once you have a matcher, use it to match against the platform declared by a // component, typically from an image or runtime. Since extracting an images // platform is a little more involved, we'll use an example against the // platform default: // -// if ok := m.Match(Default()); !ok { /* doesn't match */ } +// if ok := m.Match(Default()); !ok { /* doesn't match */ } // // This can be composed in loops for resolving runtimes or used as a filter for // fetch and select images. // // More details of the specifier syntax and platform spec follow. // -// Declaring Platform Support +// # Declaring Platform Support // // Components that have strict platform requirements should use the OCI // platform specification to declare their support. Typically, this will be // images and runtimes that should make these declaring which platform they // support specifically. This looks roughly as follows: // -// type Platform struct { -// Architecture string -// OS string -// Variant string -// } +// type Platform struct { +// Architecture string +// OS string +// Variant string +// } // // Most images and runtimes should at least set Architecture and OS, according // to their GOARCH and GOOS values, respectively (follow the OCI image // specification when in doubt). ARM should set variant under certain // discussions, which are outlined below. // -// Platform Specifiers +// # Platform Specifiers // // While the OCI platform specifications provide a tool for components to // specify structured information, user input typically doesn't need the full @@ -77,7 +77,7 @@ // where the architecture may be known but a runtime may support images from // different operating systems. // -// Normalization +// # Normalization // // Because not all users are familiar with the way the Go runtime represents // platforms, several normalizations have been provided to make this package @@ -85,17 +85,17 @@ // // The following are performed for architectures: // -// Value Normalized -// aarch64 arm64 -// armhf arm -// armel arm/v6 -// i386 386 -// x86_64 amd64 -// x86-64 amd64 +// Value Normalized +// aarch64 arm64 +// armhf arm +// armel arm/v6 +// i386 386 +// x86_64 amd64 +// x86-64 amd64 // // We also normalize the operating system `macos` to `darwin`. // -// ARM Support +// # ARM Support // // To qualify ARM architecture, the Variant field is used to qualify the arm // version. The most common arm version, v7, is represented without the variant @@ -189,8 +189,8 @@ func Parse(specifier string) (specs.Platform, error) { if isKnownOS(p.OS) { // picks a default architecture p.Architecture = runtime.GOARCH - if p.Architecture == "arm" && cpuVariant != "v7" { - p.Variant = cpuVariant + if p.Architecture == "arm" && cpuVariant() != "v7" { + p.Variant = cpuVariant() } return p, nil diff --git a/vendor/github.com/containerd/containerd/reference/reference.go b/vendor/github.com/containerd/containerd/reference/reference.go index 562ab0d49..a4bf6da60 100644 --- a/vendor/github.com/containerd/containerd/reference/reference.go +++ b/vendor/github.com/containerd/containerd/reference/reference.go @@ -85,6 +85,10 @@ var splitRe = regexp.MustCompile(`[:@]`) // Parse parses the string into a structured ref. func Parse(s string) (Spec, error) { + if strings.Contains(s, "://") { + return Spec{}, ErrInvalid + } + u, err := url.Parse("dummy://" + s) if err != nil { return Spec{}, err diff --git a/vendor/github.com/containerd/containerd/remotes/docker/auth/fetch.go b/vendor/github.com/containerd/containerd/remotes/docker/auth/fetch.go new file mode 100644 index 000000000..8b0a87e75 --- /dev/null +++ b/vendor/github.com/containerd/containerd/remotes/docker/auth/fetch.go @@ -0,0 +1,209 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package auth + +import ( + "context" + "encoding/json" + "net/http" + "net/url" + "strings" + "time" + + "github.com/containerd/containerd/log" + remoteserrors "github.com/containerd/containerd/remotes/errors" + "github.com/containerd/containerd/version" + "github.com/pkg/errors" + "golang.org/x/net/context/ctxhttp" +) + +var ( + // ErrNoToken is returned if a request is successful but the body does not + // contain an authorization token. + ErrNoToken = errors.New("authorization server did not include a token in the response") +) + +// GenerateTokenOptions generates options for fetching a token based on a challenge +func GenerateTokenOptions(ctx context.Context, host, username, secret string, c Challenge) (TokenOptions, error) { + realm, ok := c.Parameters["realm"] + if !ok { + return TokenOptions{}, errors.New("no realm specified for token auth challenge") + } + + realmURL, err := url.Parse(realm) + if err != nil { + return TokenOptions{}, errors.Wrap(err, "invalid token auth challenge realm") + } + + to := TokenOptions{ + Realm: realmURL.String(), + Service: c.Parameters["service"], + Username: username, + Secret: secret, + } + + scope, ok := c.Parameters["scope"] + if ok { + to.Scopes = append(to.Scopes, scope) + } else { + log.G(ctx).WithField("host", host).Debug("no scope specified for token auth challenge") + } + + return to, nil +} + +// TokenOptions are options for requesting a token +type TokenOptions struct { + Realm string + Service string + Scopes []string + Username string + Secret string +} + +// OAuthTokenResponse is response from fetching token with a OAuth POST request +type OAuthTokenResponse struct { + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + ExpiresIn int `json:"expires_in"` + IssuedAt time.Time `json:"issued_at"` + Scope string `json:"scope"` +} + +// FetchTokenWithOAuth fetches a token using a POST request +func FetchTokenWithOAuth(ctx context.Context, client *http.Client, headers http.Header, clientID string, to TokenOptions) (*OAuthTokenResponse, error) { + form := url.Values{} + if len(to.Scopes) > 0 { + form.Set("scope", strings.Join(to.Scopes, " ")) + } + form.Set("service", to.Service) + form.Set("client_id", clientID) + + if to.Username == "" { + form.Set("grant_type", "refresh_token") + form.Set("refresh_token", to.Secret) + } else { + form.Set("grant_type", "password") + form.Set("username", to.Username) + form.Set("password", to.Secret) + } + + req, err := http.NewRequest("POST", to.Realm, strings.NewReader(form.Encode())) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8") + for k, v := range headers { + req.Header[k] = append(req.Header[k], v...) + } + if len(req.Header.Get("User-Agent")) == 0 { + req.Header.Set("User-Agent", "containerd/"+version.Version) + } + + resp, err := ctxhttp.Do(ctx, client, req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 400 { + return nil, errors.WithStack(remoteserrors.NewUnexpectedStatusErr(resp)) + } + + decoder := json.NewDecoder(resp.Body) + + var tr OAuthTokenResponse + if err = decoder.Decode(&tr); err != nil { + return nil, errors.Wrap(err, "unable to decode token response") + } + + if tr.AccessToken == "" { + return nil, errors.WithStack(ErrNoToken) + } + + return &tr, nil +} + +// FetchTokenResponse is response from fetching token with GET request +type FetchTokenResponse struct { + Token string `json:"token"` + AccessToken string `json:"access_token"` + ExpiresIn int `json:"expires_in"` + IssuedAt time.Time `json:"issued_at"` + RefreshToken string `json:"refresh_token"` +} + +// FetchToken fetches a token using a GET request +func FetchToken(ctx context.Context, client *http.Client, headers http.Header, to TokenOptions) (*FetchTokenResponse, error) { + req, err := http.NewRequest("GET", to.Realm, nil) + if err != nil { + return nil, err + } + + for k, v := range headers { + req.Header[k] = append(req.Header[k], v...) + } + if len(req.Header.Get("User-Agent")) == 0 { + req.Header.Set("User-Agent", "containerd/"+version.Version) + } + + reqParams := req.URL.Query() + + if to.Service != "" { + reqParams.Add("service", to.Service) + } + + for _, scope := range to.Scopes { + reqParams.Add("scope", scope) + } + + if to.Secret != "" { + req.SetBasicAuth(to.Username, to.Secret) + } + + req.URL.RawQuery = reqParams.Encode() + + resp, err := ctxhttp.Do(ctx, client, req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 400 { + return nil, errors.WithStack(remoteserrors.NewUnexpectedStatusErr(resp)) + } + + decoder := json.NewDecoder(resp.Body) + + var tr FetchTokenResponse + if err = decoder.Decode(&tr); err != nil { + return nil, errors.Wrap(err, "unable to decode token response") + } + + // `access_token` is equivalent to `token` and if both are specified + // the choice is undefined. Canonicalize `access_token` by sticking + // things in `token`. + if tr.AccessToken != "" { + tr.Token = tr.AccessToken + } + + if tr.Token == "" { + return nil, errors.WithStack(ErrNoToken) + } + + return &tr, nil +} diff --git a/vendor/github.com/containerd/containerd/remotes/docker/auth.go b/vendor/github.com/containerd/containerd/remotes/docker/auth/parse.go similarity index 81% rename from vendor/github.com/containerd/containerd/remotes/docker/auth.go rename to vendor/github.com/containerd/containerd/remotes/docker/auth/parse.go index 70cfdea4f..e4529a776 100644 --- a/vendor/github.com/containerd/containerd/remotes/docker/auth.go +++ b/vendor/github.com/containerd/containerd/remotes/docker/auth/parse.go @@ -14,7 +14,7 @@ limitations under the License. */ -package docker +package auth import ( "net/http" @@ -22,31 +22,35 @@ import ( "strings" ) -type authenticationScheme byte +// AuthenticationScheme defines scheme of the authentication method +type AuthenticationScheme byte const ( - basicAuth authenticationScheme = 1 << iota // Defined in RFC 7617 - digestAuth // Defined in RFC 7616 - bearerAuth // Defined in RFC 6750 + // BasicAuth is scheme for Basic HTTP Authentication RFC 7617 + BasicAuth AuthenticationScheme = 1 << iota + // DigestAuth is scheme for HTTP Digest Access Authentication RFC 7616 + DigestAuth + // BearerAuth is scheme for OAuth 2.0 Bearer Tokens RFC 6750 + BearerAuth ) -// challenge carries information from a WWW-Authenticate response header. +// Challenge carries information from a WWW-Authenticate response header. // See RFC 2617. -type challenge struct { +type Challenge struct { // scheme is the auth-scheme according to RFC 2617 - scheme authenticationScheme + Scheme AuthenticationScheme // parameters are the auth-params according to RFC 2617 - parameters map[string]string + Parameters map[string]string } -type byScheme []challenge +type byScheme []Challenge func (bs byScheme) Len() int { return len(bs) } func (bs byScheme) Swap(i, j int) { bs[i], bs[j] = bs[j], bs[i] } // Sort in priority order: token > digest > basic -func (bs byScheme) Less(i, j int) bool { return bs[i].scheme > bs[j].scheme } +func (bs byScheme) Less(i, j int) bool { return bs[i].Scheme > bs[j].Scheme } // Octet types from RFC 2616. type octetType byte @@ -90,22 +94,23 @@ func init() { } } -func parseAuthHeader(header http.Header) []challenge { - challenges := []challenge{} +// ParseAuthHeader parses challenges from WWW-Authenticate header +func ParseAuthHeader(header http.Header) []Challenge { + challenges := []Challenge{} for _, h := range header[http.CanonicalHeaderKey("WWW-Authenticate")] { v, p := parseValueAndParams(h) - var s authenticationScheme + var s AuthenticationScheme switch v { case "basic": - s = basicAuth + s = BasicAuth case "digest": - s = digestAuth + s = DigestAuth case "bearer": - s = bearerAuth + s = BearerAuth default: continue } - challenges = append(challenges, challenge{scheme: s, parameters: p}) + challenges = append(challenges, Challenge{Scheme: s, Parameters: p}) } sort.Stable(byScheme(challenges)) return challenges @@ -129,9 +134,6 @@ func parseValueAndParams(header string) (value string, params map[string]string) } var pvalue string pvalue, s = expectTokenOrQuoted(s[1:]) - if pvalue == "" { - return - } pkey = strings.ToLower(pkey) params[pkey] = pvalue s = skipSpace(s) diff --git a/vendor/github.com/containerd/containerd/remotes/docker/authorizer.go b/vendor/github.com/containerd/containerd/remotes/docker/authorizer.go index 2f459b70c..67e4aea8d 100644 --- a/vendor/github.com/containerd/containerd/remotes/docker/authorizer.go +++ b/vendor/github.com/containerd/containerd/remotes/docker/authorizer.go @@ -19,22 +19,17 @@ package docker import ( "context" "encoding/base64" - "encoding/json" "fmt" - "io" - "io/ioutil" "net/http" - "net/url" "strings" "sync" - "time" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/log" - "github.com/containerd/containerd/version" + "github.com/containerd/containerd/remotes/docker/auth" + remoteerrors "github.com/containerd/containerd/remotes/errors" "github.com/pkg/errors" "github.com/sirupsen/logrus" - "golang.org/x/net/context/ctxhttp" ) type dockerAuthorizer struct { @@ -136,8 +131,8 @@ func (a *dockerAuthorizer) AddResponses(ctx context.Context, responses []*http.R a.mu.Lock() defer a.mu.Unlock() - for _, c := range parseAuthHeader(last.Header) { - if c.scheme == bearerAuth { + for _, c := range auth.ParseAuthHeader(last.Header) { + if c.Scheme == auth.BearerAuth { if err := invalidAuthorization(c, responses); err != nil { delete(a.handlers, host) return err @@ -153,26 +148,35 @@ func (a *dockerAuthorizer) AddResponses(ctx context.Context, responses []*http.R return nil } - common, err := a.generateTokenOptions(ctx, host, c) + var username, secret string + if a.credentials != nil { + var err error + username, secret, err = a.credentials(host) + if err != nil { + return err + } + } + + common, err := auth.GenerateTokenOptions(ctx, host, username, secret, c) if err != nil { return err } - a.handlers[host] = newAuthHandler(a.client, a.header, c.scheme, common) + a.handlers[host] = newAuthHandler(a.client, a.header, c.Scheme, common) return nil - } else if c.scheme == basicAuth && a.credentials != nil { + } else if c.Scheme == auth.BasicAuth && a.credentials != nil { username, secret, err := a.credentials(host) if err != nil { return err } if username != "" && secret != "" { - common := tokenOptions{ - username: username, - secret: secret, + common := auth.TokenOptions{ + Username: username, + Secret: secret, } - a.handlers[host] = newAuthHandler(a.client, a.header, c.scheme, common) + a.handlers[host] = newAuthHandler(a.client, a.header, c.Scheme, common) return nil } } @@ -180,38 +184,6 @@ func (a *dockerAuthorizer) AddResponses(ctx context.Context, responses []*http.R return errors.Wrap(errdefs.ErrNotImplemented, "failed to find supported auth scheme") } -func (a *dockerAuthorizer) generateTokenOptions(ctx context.Context, host string, c challenge) (tokenOptions, error) { - realm, ok := c.parameters["realm"] - if !ok { - return tokenOptions{}, errors.New("no realm specified for token auth challenge") - } - - realmURL, err := url.Parse(realm) - if err != nil { - return tokenOptions{}, errors.Wrap(err, "invalid token auth challenge realm") - } - - to := tokenOptions{ - realm: realmURL.String(), - service: c.parameters["service"], - } - - scope, ok := c.parameters["scope"] - if ok { - to.scopes = append(to.scopes, scope) - } else { - log.G(ctx).WithField("host", host).Debug("no scope specified for token auth challenge") - } - - if a.credentials != nil { - to.username, to.secret, err = a.credentials(host) - if err != nil { - return tokenOptions{}, err - } - } - return to, nil -} - // authResult is used to control limit rate. type authResult struct { sync.WaitGroup @@ -228,17 +200,17 @@ type authHandler struct { client *http.Client // only support basic and bearer schemes - scheme authenticationScheme + scheme auth.AuthenticationScheme // common contains common challenge answer - common tokenOptions + common auth.TokenOptions // scopedTokens caches token indexed by scopes, which used in // bearer auth case scopedTokens map[string]*authResult } -func newAuthHandler(client *http.Client, hdr http.Header, scheme authenticationScheme, opts tokenOptions) *authHandler { +func newAuthHandler(client *http.Client, hdr http.Header, scheme auth.AuthenticationScheme, opts auth.TokenOptions) *authHandler { return &authHandler{ header: hdr, client: client, @@ -250,17 +222,17 @@ func newAuthHandler(client *http.Client, hdr http.Header, scheme authenticationS func (ah *authHandler) authorize(ctx context.Context) (string, error) { switch ah.scheme { - case basicAuth: + case auth.BasicAuth: return ah.doBasicAuth(ctx) - case bearerAuth: + case auth.BearerAuth: return ah.doBearerAuth(ctx) default: - return "", errors.Wrap(errdefs.ErrNotImplemented, "failed to find supported auth scheme") + return "", errors.Wrapf(errdefs.ErrNotImplemented, "failed to find supported auth scheme: %s", string(ah.scheme)) } } func (ah *authHandler) doBasicAuth(ctx context.Context) (string, error) { - username, secret := ah.common.username, ah.common.secret + username, secret := ah.common.Username, ah.common.Secret if username == "" || secret == "" { return "", fmt.Errorf("failed to handle basic auth because missing username or secret") @@ -270,14 +242,14 @@ func (ah *authHandler) doBasicAuth(ctx context.Context) (string, error) { return fmt.Sprintf("Basic %s", auth), nil } -func (ah *authHandler) doBearerAuth(ctx context.Context) (string, error) { +func (ah *authHandler) doBearerAuth(ctx context.Context) (token string, err error) { // copy common tokenOptions to := ah.common - to.scopes = GetTokenScopes(ctx, to.scopes) + to.Scopes = GetTokenScopes(ctx, to.Scopes) // Docs: https://docs.docker.com/registry/spec/auth/scope - scoped := strings.Join(to.scopes, " ") + scoped := strings.Join(to.Scopes, " ") ah.Lock() if r, exist := ah.scopedTokens[scoped]; exist { @@ -292,180 +264,52 @@ func (ah *authHandler) doBearerAuth(ctx context.Context) (string, error) { ah.scopedTokens[scoped] = r ah.Unlock() + defer func() { + token = fmt.Sprintf("Bearer %s", token) + r.token, r.err = token, err + r.Done() + }() + // fetch token for the resource scope - var ( - token string - err error - ) - if to.secret != "" { + if to.Secret != "" { + defer func() { + err = errors.Wrap(err, "failed to fetch oauth token") + }() // credential information is provided, use oauth POST endpoint - token, err = ah.fetchTokenWithOAuth(ctx, to) - err = errors.Wrap(err, "failed to fetch oauth token") - } else { - // do request anonymously - token, err = ah.fetchToken(ctx, to) - err = errors.Wrap(err, "failed to fetch anonymous token") - } - token = fmt.Sprintf("Bearer %s", token) - - r.token, r.err = token, err - r.Done() - return r.token, r.err -} - -type tokenOptions struct { - realm string - service string - scopes []string - username string - secret string -} - -type postTokenResponse struct { - AccessToken string `json:"access_token"` - RefreshToken string `json:"refresh_token"` - ExpiresIn int `json:"expires_in"` - IssuedAt time.Time `json:"issued_at"` - Scope string `json:"scope"` -} - -func (ah *authHandler) fetchTokenWithOAuth(ctx context.Context, to tokenOptions) (string, error) { - form := url.Values{} - if len(to.scopes) > 0 { - form.Set("scope", strings.Join(to.scopes, " ")) - } - form.Set("service", to.service) - // TODO: Allow setting client_id - form.Set("client_id", "containerd-client") - - if to.username == "" { - form.Set("grant_type", "refresh_token") - form.Set("refresh_token", to.secret) - } else { - form.Set("grant_type", "password") - form.Set("username", to.username) - form.Set("password", to.secret) - } - - req, err := http.NewRequest("POST", to.realm, strings.NewReader(form.Encode())) - if err != nil { - return "", err - } - req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8") - if ah.header != nil { - for k, v := range ah.header { - req.Header[k] = append(req.Header[k], v...) + // TODO: Allow setting client_id + resp, err := auth.FetchTokenWithOAuth(ctx, ah.client, ah.header, "containerd-client", to) + if err != nil { + var errStatus remoteerrors.ErrUnexpectedStatus + if errors.As(err, &errStatus) { + // Registries without support for POST may return 404 for POST /v2/token. + // As of September 2017, GCR is known to return 404. + // As of February 2018, JFrog Artifactory is known to return 401. + if (errStatus.StatusCode == 405 && to.Username != "") || errStatus.StatusCode == 404 || errStatus.StatusCode == 401 { + resp, err := auth.FetchToken(ctx, ah.client, ah.header, to) + if err != nil { + return "", err + } + return resp.Token, nil + } + log.G(ctx).WithFields(logrus.Fields{ + "status": errStatus.Status, + "body": string(errStatus.Body), + }).Debugf("token request failed") + } + return "", err } + return resp.AccessToken, nil } - if len(req.Header.Get("User-Agent")) == 0 { - req.Header.Set("User-Agent", "containerd/"+version.Version) - } - - resp, err := ctxhttp.Do(ctx, ah.client, req) + // do request anonymously + resp, err := auth.FetchToken(ctx, ah.client, ah.header, to) if err != nil { - return "", err + return "", errors.Wrap(err, "failed to fetch anonymous token") } - defer resp.Body.Close() - - // Registries without support for POST may return 404 for POST /v2/token. - // As of September 2017, GCR is known to return 404. - // As of February 2018, JFrog Artifactory is known to return 401. - if (resp.StatusCode == 405 && to.username != "") || resp.StatusCode == 404 || resp.StatusCode == 401 { - return ah.fetchToken(ctx, to) - } else if resp.StatusCode < 200 || resp.StatusCode >= 400 { - b, _ := ioutil.ReadAll(io.LimitReader(resp.Body, 64000)) // 64KB - log.G(ctx).WithFields(logrus.Fields{ - "status": resp.Status, - "body": string(b), - }).Debugf("token request failed") - // TODO: handle error body and write debug output - return "", errors.Errorf("unexpected status: %s", resp.Status) - } - - decoder := json.NewDecoder(resp.Body) - - var tr postTokenResponse - if err = decoder.Decode(&tr); err != nil { - return "", fmt.Errorf("unable to decode token response: %s", err) - } - - return tr.AccessToken, nil + return resp.Token, nil } -type getTokenResponse struct { - Token string `json:"token"` - AccessToken string `json:"access_token"` - ExpiresIn int `json:"expires_in"` - IssuedAt time.Time `json:"issued_at"` - RefreshToken string `json:"refresh_token"` -} - -// fetchToken fetches a token using a GET request -func (ah *authHandler) fetchToken(ctx context.Context, to tokenOptions) (string, error) { - req, err := http.NewRequest("GET", to.realm, nil) - if err != nil { - return "", err - } - - if ah.header != nil { - for k, v := range ah.header { - req.Header[k] = append(req.Header[k], v...) - } - } - if len(req.Header.Get("User-Agent")) == 0 { - req.Header.Set("User-Agent", "containerd/"+version.Version) - } - - reqParams := req.URL.Query() - - if to.service != "" { - reqParams.Add("service", to.service) - } - - for _, scope := range to.scopes { - reqParams.Add("scope", scope) - } - - if to.secret != "" { - req.SetBasicAuth(to.username, to.secret) - } - - req.URL.RawQuery = reqParams.Encode() - - resp, err := ctxhttp.Do(ctx, ah.client, req) - if err != nil { - return "", err - } - defer resp.Body.Close() - - if resp.StatusCode < 200 || resp.StatusCode >= 400 { - // TODO: handle error body and write debug output - return "", errors.Errorf("unexpected status: %s", resp.Status) - } - - decoder := json.NewDecoder(resp.Body) - - var tr getTokenResponse - if err = decoder.Decode(&tr); err != nil { - return "", fmt.Errorf("unable to decode token response: %s", err) - } - - // `access_token` is equivalent to `token` and if both are specified - // the choice is undefined. Canonicalize `access_token` by sticking - // things in `token`. - if tr.AccessToken != "" { - tr.Token = tr.AccessToken - } - - if tr.Token == "" { - return "", ErrNoToken - } - - return tr.Token, nil -} - -func invalidAuthorization(c challenge, responses []*http.Response) error { - errStr := c.parameters["error"] +func invalidAuthorization(c auth.Challenge, responses []*http.Response) error { + errStr := c.Parameters["error"] if errStr == "" { return nil } diff --git a/vendor/github.com/containerd/containerd/remotes/docker/errcode.go b/vendor/github.com/containerd/containerd/remotes/docker/errcode.go index 7d1e54f2a..8c623bcbe 100644 --- a/vendor/github.com/containerd/containerd/remotes/docker/errcode.go +++ b/vendor/github.com/containerd/containerd/remotes/docker/errcode.go @@ -163,7 +163,7 @@ type ErrorDescriptor struct { // keyed value when serializing api errors. Value string - // Message is a short, human readable decription of the error condition + // Message is a short, human readable description of the error condition // included in API responses. Message string diff --git a/vendor/github.com/containerd/containerd/remotes/docker/fetcher.go b/vendor/github.com/containerd/containerd/remotes/docker/fetcher.go index 5aaaf9e2a..4b2c10e9a 100644 --- a/vendor/github.com/containerd/containerd/remotes/docker/fetcher.go +++ b/vendor/github.com/containerd/containerd/remotes/docker/fetcher.go @@ -45,7 +45,7 @@ func (r dockerFetcher) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.R return nil, errors.Wrap(errdefs.ErrNotFound, "no pull hosts") } - ctx, err := contextWithRepositoryScope(ctx, r.refspec, false) + ctx, err := ContextWithRepositoryScope(ctx, r.refspec, false) if err != nil { return nil, err } diff --git a/vendor/github.com/containerd/containerd/remotes/docker/httpreadseeker.go b/vendor/github.com/containerd/containerd/remotes/docker/httpreadseeker.go index 9175b6a7a..58c866bcd 100644 --- a/vendor/github.com/containerd/containerd/remotes/docker/httpreadseeker.go +++ b/vendor/github.com/containerd/containerd/remotes/docker/httpreadseeker.go @@ -26,12 +26,16 @@ import ( "github.com/pkg/errors" ) +const maxRetry = 3 + type httpReadSeeker struct { size int64 offset int64 rc io.ReadCloser open func(offset int64) (io.ReadCloser, error) closed bool + + errsWithNoProgress int } func newHTTPReadSeeker(size int64, open func(offset int64) (io.ReadCloser, error)) (io.ReadCloser, error) { @@ -53,6 +57,27 @@ func (hrs *httpReadSeeker) Read(p []byte) (n int, err error) { n, err = rd.Read(p) hrs.offset += int64(n) + if n > 0 || err == nil { + hrs.errsWithNoProgress = 0 + } + if err == io.ErrUnexpectedEOF { + // connection closed unexpectedly. try reconnecting. + if n == 0 { + hrs.errsWithNoProgress++ + if hrs.errsWithNoProgress > maxRetry { + return // too many retries for this offset with no progress + } + } + if hrs.rc != nil { + if clsErr := hrs.rc.Close(); clsErr != nil { + log.L.WithError(clsErr).Errorf("httpReadSeeker: failed to close ReadCloser") + } + hrs.rc = nil + } + if _, err2 := hrs.reader(); err2 == nil { + return n, nil + } + } return } @@ -121,7 +146,7 @@ func (hrs *httpReadSeeker) reader() (io.Reader, error) { rc, err := hrs.open(hrs.offset) if err != nil { - return nil, errors.Wrapf(err, "httpReaderSeeker: failed open") + return nil, errors.Wrapf(err, "httpReadSeeker: failed open") } if hrs.rc != nil { diff --git a/vendor/github.com/containerd/containerd/remotes/docker/pusher.go b/vendor/github.com/containerd/containerd/remotes/docker/pusher.go index d46396b7f..75be84859 100644 --- a/vendor/github.com/containerd/containerd/remotes/docker/pusher.go +++ b/vendor/github.com/containerd/containerd/remotes/docker/pusher.go @@ -23,6 +23,7 @@ import ( "net/http" "net/url" "strings" + "sync" "time" "github.com/containerd/containerd/content" @@ -30,6 +31,7 @@ import ( "github.com/containerd/containerd/images" "github.com/containerd/containerd/log" "github.com/containerd/containerd/remotes" + remoteserrors "github.com/containerd/containerd/remotes/errors" digest "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" @@ -43,17 +45,47 @@ type dockerPusher struct { tracker StatusTracker } +// Writer implements Ingester API of content store. This allows the client +// to receive ErrUnavailable when there is already an on-going upload. +// Note that the tracker MUST implement StatusTrackLocker interface to avoid +// race condition on StatusTracker. +func (p dockerPusher) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) { + var wOpts content.WriterOpts + for _, opt := range opts { + if err := opt(&wOpts); err != nil { + return nil, err + } + } + if wOpts.Ref == "" { + return nil, errors.Wrap(errdefs.ErrInvalidArgument, "ref must not be empty") + } + return p.push(ctx, wOpts.Desc, wOpts.Ref, true) +} + func (p dockerPusher) Push(ctx context.Context, desc ocispec.Descriptor) (content.Writer, error) { - ctx, err := contextWithRepositoryScope(ctx, p.refspec, true) + return p.push(ctx, desc, remotes.MakeRefKey(ctx, desc), false) +} + +func (p dockerPusher) push(ctx context.Context, desc ocispec.Descriptor, ref string, unavailableOnFail bool) (content.Writer, error) { + if l, ok := p.tracker.(StatusTrackLocker); ok { + l.Lock(ref) + defer l.Unlock(ref) + } + ctx, err := ContextWithRepositoryScope(ctx, p.refspec, true) if err != nil { return nil, err } - ref := remotes.MakeRefKey(ctx, desc) status, err := p.tracker.GetStatus(ref) if err == nil { - if status.Offset == status.Total { + if status.Committed && status.Offset == status.Total { return nil, errors.Wrapf(errdefs.ErrAlreadyExists, "ref %v", ref) } + if unavailableOnFail && status.ErrClosed == nil { + // Another push of this ref is happening elsewhere. The rest of function + // will continue only when `errdefs.IsNotFound(err) == true` (i.e. there + // is no actively-tracked ref already). + return nil, errors.Wrap(errdefs.ErrUnavailable, "push is on-going") + } // TODO: Handle incomplete status } else if !errdefs.IsNotFound(err) { return nil, errors.Wrap(err, "failed to get status") @@ -104,8 +136,11 @@ func (p dockerPusher) Push(ctx context.Context, desc ocispec.Descriptor) (conten if exists { p.tracker.SetStatus(ref, Status{ + Committed: true, Status: content.Status{ - Ref: ref, + Ref: ref, + Total: desc.Size, + Offset: desc.Size, // TODO: Set updated time? }, }) @@ -113,9 +148,10 @@ func (p dockerPusher) Push(ctx context.Context, desc ocispec.Descriptor) (conten return nil, errors.Wrapf(errdefs.ErrAlreadyExists, "content %v on remote", desc.Digest) } } else if resp.StatusCode != http.StatusNotFound { + err := remoteserrors.NewUnexpectedStatusErr(resp) + log.G(ctx).WithField("resp", resp).WithField("body", string(err.(remoteserrors.ErrUnexpectedStatus).Body)).Debug("unexpected response") resp.Body.Close() - // TODO: log error - return nil, errors.Errorf("unexpected response: %s", resp.Status) + return nil, err } resp.Body.Close() } @@ -131,7 +167,7 @@ func (p dockerPusher) Push(ctx context.Context, desc ocispec.Descriptor) (conten var resp *http.Response if fromRepo := selectRepositoryMountCandidate(p.refspec, desc.Annotations); fromRepo != "" { preq := requestWithMountFrom(req, desc.Digest.String(), fromRepo) - pctx := contextWithAppendPullRepositoryScope(ctx, fromRepo) + pctx := ContextWithAppendPullRepositoryScope(ctx, fromRepo) // NOTE: the fromRepo might be private repo and // auth service still can grant token without error. @@ -164,14 +200,18 @@ func (p dockerPusher) Push(ctx context.Context, desc ocispec.Descriptor) (conten case http.StatusOK, http.StatusAccepted, http.StatusNoContent: case http.StatusCreated: p.tracker.SetStatus(ref, Status{ + Committed: true, Status: content.Status{ - Ref: ref, + Ref: ref, + Total: desc.Size, + Offset: desc.Size, }, }) return nil, errors.Wrapf(errdefs.ErrAlreadyExists, "content %v on remote", desc.Digest) default: - // TODO: log error - return nil, errors.Errorf("unexpected response: %s", resp.Status) + err := remoteserrors.NewUnexpectedStatusErr(resp) + log.G(ctx).WithField("resp", resp).WithField("body", string(err.(remoteserrors.ErrUnexpectedStatus).Body)).Debug("unexpected response") + return nil, err } var ( @@ -222,47 +262,35 @@ func (p dockerPusher) Push(ctx context.Context, desc ocispec.Descriptor) (conten // TODO: Support chunked upload - pr, pw := io.Pipe() - respC := make(chan *http.Response, 1) - body := ioutil.NopCloser(pr) + pushw := newPushWriter(p.dockerBase, ref, desc.Digest, p.tracker, isManifest) req.body = func() (io.ReadCloser, error) { - if body == nil { - return nil, errors.New("cannot reuse body, request must be retried") - } - // Only use the body once since pipe cannot be seeked - ob := body - body = nil - return ob, nil + pr, pw := io.Pipe() + pushw.setPipe(pw) + return ioutil.NopCloser(pr), nil } req.size = desc.Size go func() { - defer close(respC) resp, err := req.doWithRetries(ctx, nil) if err != nil { - pr.CloseWithError(err) + pushw.setError(err) + pushw.Close() return } switch resp.StatusCode { case http.StatusOK, http.StatusCreated, http.StatusNoContent: default: - // TODO: log error - pr.CloseWithError(errors.Errorf("unexpected response: %s", resp.Status)) + err := remoteserrors.NewUnexpectedStatusErr(resp) + log.G(ctx).WithField("resp", resp).WithField("body", string(err.(remoteserrors.ErrUnexpectedStatus).Body)).Debug("unexpected response") + pushw.setError(err) + pushw.Close() } - respC <- resp + pushw.setResponse(resp) }() - return &pushWriter{ - base: p.dockerBase, - ref: ref, - pipe: pw, - responseC: respC, - isManifest: isManifest, - expected: desc.Digest, - tracker: p.tracker, - }, nil + return pushw, nil } func getManifestPath(object string, dgst digest.Digest) []string { @@ -288,19 +316,77 @@ type pushWriter struct { base *dockerBase ref string - pipe *io.PipeWriter - responseC <-chan *http.Response + pipe *io.PipeWriter + + pipeC chan *io.PipeWriter + respC chan *http.Response + closeOnce sync.Once + errC chan error + isManifest bool expected digest.Digest tracker StatusTracker } +func newPushWriter(db *dockerBase, ref string, expected digest.Digest, tracker StatusTracker, isManifest bool) *pushWriter { + // Initialize and create response + return &pushWriter{ + base: db, + ref: ref, + expected: expected, + tracker: tracker, + pipeC: make(chan *io.PipeWriter, 1), + respC: make(chan *http.Response, 1), + errC: make(chan error, 1), + isManifest: isManifest, + } +} + +func (pw *pushWriter) setPipe(p *io.PipeWriter) { + pw.pipeC <- p +} + +func (pw *pushWriter) setError(err error) { + pw.errC <- err +} +func (pw *pushWriter) setResponse(resp *http.Response) { + pw.respC <- resp +} + func (pw *pushWriter) Write(p []byte) (n int, err error) { status, err := pw.tracker.GetStatus(pw.ref) if err != nil { return n, err } + + if pw.pipe == nil { + p, ok := <-pw.pipeC + if !ok { + return 0, io.ErrClosedPipe + } + pw.pipe = p + } else { + select { + case p, ok := <-pw.pipeC: + if !ok { + return 0, io.ErrClosedPipe + } + pw.pipe.CloseWithError(content.ErrReset) + pw.pipe = p + + // If content has already been written, the bytes + // cannot be written and the caller must reset + if status.Offset > 0 { + status.Offset = 0 + status.UpdatedAt = time.Now() + pw.tracker.SetStatus(pw.ref, status) + return 0, content.ErrReset + } + default: + } + } + n, err = pw.pipe.Write(p) status.Offset += int64(n) status.UpdatedAt = time.Now() @@ -309,7 +395,21 @@ func (pw *pushWriter) Write(p []byte) (n int, err error) { } func (pw *pushWriter) Close() error { - return pw.pipe.Close() + // Ensure pipeC is closed but handle `Close()` being + // called multiple times without panicking + pw.closeOnce.Do(func() { + close(pw.pipeC) + }) + if pw.pipe != nil { + status, err := pw.tracker.GetStatus(pw.ref) + if err == nil && !status.Committed { + // Closing an incomplete writer. Record this as an error so that following write can retry it. + status.ErrClosed = errors.New("closed incomplete writer") + pw.tracker.SetStatus(pw.ref, status) + } + return pw.pipe.Close() + } + return nil } func (pw *pushWriter) Status() (content.Status, error) { @@ -335,21 +435,44 @@ func (pw *pushWriter) Commit(ctx context.Context, size int64, expected digest.Di if err := pw.pipe.Close(); err != nil { return err } - // TODO: Update status to determine committing - // TODO: timeout waiting for response - resp := <-pw.responseC - if resp == nil { - return errors.New("no response") + var resp *http.Response + select { + case err := <-pw.errC: + if err != nil { + return err + } + case resp = <-pw.respC: + defer resp.Body.Close() + case p, ok := <-pw.pipeC: + // check whether the pipe has changed in the commit, because sometimes Write + // can complete successfully, but the pipe may have changed. In that case, the + // content needs to be reset. + if !ok { + return io.ErrClosedPipe + } + pw.pipe.CloseWithError(content.ErrReset) + pw.pipe = p + status, err := pw.tracker.GetStatus(pw.ref) + if err != nil { + return err + } + // If content has already been written, the bytes + // cannot be written again and the caller must reset + if status.Offset > 0 { + status.Offset = 0 + status.UpdatedAt = time.Now() + pw.tracker.SetStatus(pw.ref, status) + return content.ErrReset + } } - defer resp.Body.Close() // 201 is specified return status, some registries return // 200, 202 or 204. switch resp.StatusCode { case http.StatusOK, http.StatusCreated, http.StatusNoContent, http.StatusAccepted: default: - return errors.Errorf("unexpected status: %s", resp.Status) + return remoteserrors.NewUnexpectedStatusErr(resp) } status, err := pw.tracker.GetStatus(pw.ref) @@ -374,6 +497,10 @@ func (pw *pushWriter) Commit(ctx context.Context, size int64, expected digest.Di return errors.Errorf("got digest %s, expected %s", actual, expected) } + status.Committed = true + status.UpdatedAt = time.Now() + pw.tracker.SetStatus(pw.ref, status) + return nil } diff --git a/vendor/github.com/containerd/containerd/remotes/docker/registry.go b/vendor/github.com/containerd/containerd/remotes/docker/registry.go index 7c231d928..1e77d4c86 100644 --- a/vendor/github.com/containerd/containerd/remotes/docker/registry.go +++ b/vendor/github.com/containerd/containerd/remotes/docker/registry.go @@ -17,7 +17,10 @@ package docker import ( + "net" "net/http" + + "github.com/pkg/errors" ) // HostCapabilities represent the capabilities of the registry @@ -56,6 +59,7 @@ const ( // Reserved for future capabilities (i.e. search, catalog, remove) ) +// Has checks whether the capabilities list has the provide capability func (c HostCapabilities) Has(t HostCapabilities) bool { return c&t == t } @@ -201,12 +205,41 @@ func MatchAllHosts(string) (bool, error) { // MatchLocalhost is a host match function which returns true for // localhost. +// +// Note: this does not handle matching of ip addresses in octal, +// decimal or hex form. func MatchLocalhost(host string) (bool, error) { - for _, s := range []string{"localhost", "127.0.0.1", "[::1]"} { - if len(host) >= len(s) && host[0:len(s)] == s && (len(host) == len(s) || host[len(s)] == ':') { - return true, nil - } + switch { + case host == "::1": + return true, nil + case host == "[::1]": + return true, nil } - return host == "::1", nil + h, p, err := net.SplitHostPort(host) + // addrError helps distinguish between errors of form + // "no colon in address" and "too many colons in address". + // The former is fine as the host string need not have a + // port. Latter needs to be handled. + addrError := &net.AddrError{ + Err: "missing port in address", + Addr: host, + } + if err != nil { + if err.Error() != addrError.Error() { + return false, err + } + // host string without any port specified + h = host + } else if len(p) == 0 { + return false, errors.New("invalid host name format") + } + + // use ipv4 dotted decimal for further checking + if h == "localhost" { + h = "127.0.0.1" + } + ip := net.ParseIP(h) + + return ip.IsLoopback(), nil } diff --git a/vendor/github.com/containerd/containerd/remotes/docker/resolver.go b/vendor/github.com/containerd/containerd/remotes/docker/resolver.go index 40831f168..1be9e1d05 100644 --- a/vendor/github.com/containerd/containerd/remotes/docker/resolver.go +++ b/vendor/github.com/containerd/containerd/remotes/docker/resolver.go @@ -41,10 +41,6 @@ import ( ) var ( - // ErrNoToken is returned if a request is successful but the body does not - // contain an authorization token. - ErrNoToken = errors.New("authorization server did not include a token in the response") - // ErrInvalidAuthorization is used when credentials are passed to a server but // those credentials are rejected. ErrInvalidAuthorization = errors.New("authorization failed") @@ -223,20 +219,15 @@ func (r *countingReader) Read(p []byte) (int, error) { var _ remotes.Resolver = &dockerResolver{} func (r *dockerResolver) Resolve(ctx context.Context, ref string) (string, ocispec.Descriptor, error) { - refspec, err := reference.Parse(ref) + base, err := r.resolveDockerBase(ref) if err != nil { return "", ocispec.Descriptor{}, err } - + refspec := base.refspec if refspec.Object == "" { return "", ocispec.Descriptor{}, reference.ErrObjectRequired } - base, err := r.base(refspec) - if err != nil { - return "", ocispec.Descriptor{}, err - } - var ( firstErr error paths [][]string @@ -267,7 +258,7 @@ func (r *dockerResolver) Resolve(ctx context.Context, ref string) (string, ocisp return "", ocispec.Descriptor{}, errors.Wrap(errdefs.ErrNotFound, "no resolve hosts") } - ctx, err = contextWithRepositoryScope(ctx, refspec, false) + ctx, err = ContextWithRepositoryScope(ctx, refspec, false) if err != nil { return "", ocispec.Descriptor{}, err } @@ -404,12 +395,7 @@ func (r *dockerResolver) Resolve(ctx context.Context, ref string) (string, ocisp } func (r *dockerResolver) Fetcher(ctx context.Context, ref string) (remotes.Fetcher, error) { - refspec, err := reference.Parse(ref) - if err != nil { - return nil, err - } - - base, err := r.base(refspec) + base, err := r.resolveDockerBase(ref) if err != nil { return nil, err } @@ -420,23 +406,27 @@ func (r *dockerResolver) Fetcher(ctx context.Context, ref string) (remotes.Fetch } func (r *dockerResolver) Pusher(ctx context.Context, ref string) (remotes.Pusher, error) { - refspec, err := reference.Parse(ref) - if err != nil { - return nil, err - } - - base, err := r.base(refspec) + base, err := r.resolveDockerBase(ref) if err != nil { return nil, err } return dockerPusher{ dockerBase: base, - object: refspec.Object, + object: base.refspec.Object, tracker: r.tracker, }, nil } +func (r *dockerResolver) resolveDockerBase(ref string) (*dockerBase, error) { + refspec, err := reference.Parse(ref) + if err != nil { + return nil, err + } + + return r.base(refspec) +} + type dockerBase struct { refspec reference.Spec repository string @@ -468,10 +458,11 @@ func (r *dockerBase) filterHosts(caps HostCapabilities) (hosts []RegistryHost) { } func (r *dockerBase) request(host RegistryHost, method string, ps ...string) *request { - header := http.Header{} - for key, value := range r.header { - header[key] = append(header[key], value...) + header := r.header.Clone() + if header == nil { + header = http.Header{} } + for key, value := range host.Header { header[key] = append(header[key], value...) } diff --git a/vendor/github.com/containerd/containerd/remotes/docker/scope.go b/vendor/github.com/containerd/containerd/remotes/docker/scope.go index c8541c455..fe57f023d 100644 --- a/vendor/github.com/containerd/containerd/remotes/docker/scope.go +++ b/vendor/github.com/containerd/containerd/remotes/docker/scope.go @@ -26,10 +26,10 @@ import ( "github.com/containerd/containerd/reference" ) -// repositoryScope returns a repository scope string such as "repository:foo/bar:pull" +// RepositoryScope returns a repository scope string such as "repository:foo/bar:pull" // for "host/foo/bar:baz". // When push is true, both pull and push are added to the scope. -func repositoryScope(refspec reference.Spec, push bool) (string, error) { +func RepositoryScope(refspec reference.Spec, push bool) (string, error) { u, err := url.Parse("dummy://" + refspec.Locator) if err != nil { return "", err @@ -45,9 +45,9 @@ func repositoryScope(refspec reference.Spec, push bool) (string, error) { // value: []string (e.g. {"registry:foo/bar:pull"}) type tokenScopesKey struct{} -// contextWithRepositoryScope returns a context with tokenScopesKey{} and the repository scope value. -func contextWithRepositoryScope(ctx context.Context, refspec reference.Spec, push bool) (context.Context, error) { - s, err := repositoryScope(refspec, push) +// ContextWithRepositoryScope returns a context with tokenScopesKey{} and the repository scope value. +func ContextWithRepositoryScope(ctx context.Context, refspec reference.Spec, push bool) (context.Context, error) { + s, err := RepositoryScope(refspec, push) if err != nil { return nil, err } @@ -66,9 +66,9 @@ func WithScope(ctx context.Context, scope string) context.Context { return context.WithValue(ctx, tokenScopesKey{}, scopes) } -// contextWithAppendPullRepositoryScope is used to append repository pull +// ContextWithAppendPullRepositoryScope is used to append repository pull // scope into existing scopes indexed by the tokenScopesKey{}. -func contextWithAppendPullRepositoryScope(ctx context.Context, repo string) context.Context { +func ContextWithAppendPullRepositoryScope(ctx context.Context, repo string) context.Context { return WithScope(ctx, fmt.Sprintf("repository:%s:pull", repo)) } diff --git a/vendor/github.com/containerd/containerd/remotes/docker/status.go b/vendor/github.com/containerd/containerd/remotes/docker/status.go index 8069d6767..08768c297 100644 --- a/vendor/github.com/containerd/containerd/remotes/docker/status.go +++ b/vendor/github.com/containerd/containerd/remotes/docker/status.go @@ -21,6 +21,7 @@ import ( "github.com/containerd/containerd/content" "github.com/containerd/containerd/errdefs" + "github.com/moby/locker" "github.com/pkg/errors" ) @@ -28,6 +29,11 @@ import ( type Status struct { content.Status + Committed bool + + // ErrClosed contains error encountered on close. + ErrClosed error + // UploadUUID is used by the Docker registry to reference blob uploads UploadUUID string } @@ -38,15 +44,24 @@ type StatusTracker interface { SetStatus(string, Status) } +// StatusTrackLocker to track status of operations with lock +type StatusTrackLocker interface { + StatusTracker + Lock(string) + Unlock(string) +} + type memoryStatusTracker struct { statuses map[string]Status m sync.Mutex + locker *locker.Locker } // NewInMemoryTracker returns a StatusTracker that tracks content status in-memory -func NewInMemoryTracker() StatusTracker { +func NewInMemoryTracker() StatusTrackLocker { return &memoryStatusTracker{ statuses: map[string]Status{}, + locker: locker.New(), } } @@ -65,3 +80,11 @@ func (t *memoryStatusTracker) SetStatus(ref string, status Status) { t.statuses[ref] = status t.m.Unlock() } + +func (t *memoryStatusTracker) Lock(ref string) { + t.locker.Lock(ref) +} + +func (t *memoryStatusTracker) Unlock(ref string) { + t.locker.Unlock(ref) +} diff --git a/vendor/github.com/containerd/containerd/remotes/errors/errors.go b/vendor/github.com/containerd/containerd/remotes/errors/errors.go new file mode 100644 index 000000000..519dbac10 --- /dev/null +++ b/vendor/github.com/containerd/containerd/remotes/errors/errors.go @@ -0,0 +1,56 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package errors + +import ( + "fmt" + "io" + "io/ioutil" + "net/http" +) + +var _ error = ErrUnexpectedStatus{} + +// ErrUnexpectedStatus is returned if a registry API request returned with unexpected HTTP status +type ErrUnexpectedStatus struct { + Status string + StatusCode int + Body []byte + RequestURL, RequestMethod string +} + +func (e ErrUnexpectedStatus) Error() string { + return fmt.Sprintf("unexpected status: %s", e.Status) +} + +// NewUnexpectedStatusErr creates an ErrUnexpectedStatus from HTTP response +func NewUnexpectedStatusErr(resp *http.Response) error { + var b []byte + if resp.Body != nil { + b, _ = ioutil.ReadAll(io.LimitReader(resp.Body, 64000)) // 64KB + } + err := ErrUnexpectedStatus{ + Body: b, + Status: resp.Status, + StatusCode: resp.StatusCode, + RequestMethod: resp.Request.Method, + } + if resp.Request.URL != nil { + err.RequestURL = resp.Request.URL.String() + } + return err +} diff --git a/vendor/github.com/containerd/containerd/remotes/handlers.go b/vendor/github.com/containerd/containerd/remotes/handlers.go index 671fea106..8f79c608e 100644 --- a/vendor/github.com/containerd/containerd/remotes/handlers.go +++ b/vendor/github.com/containerd/containerd/remotes/handlers.go @@ -31,6 +31,7 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "golang.org/x/sync/semaphore" ) type refKeyPrefix struct{} @@ -55,25 +56,32 @@ func WithMediaTypeKeyPrefix(ctx context.Context, mediaType, prefix string) conte // used to lookup ongoing processes related to the descriptor. This function // may look to the context to namespace the reference appropriately. func MakeRefKey(ctx context.Context, desc ocispec.Descriptor) string { + key := desc.Digest.String() + if desc.Annotations != nil { + if name, ok := desc.Annotations[ocispec.AnnotationRefName]; ok { + key = fmt.Sprintf("%s@%s", name, desc.Digest.String()) + } + } + if v := ctx.Value(refKeyPrefix{}); v != nil { values := v.(map[string]string) if prefix := values[desc.MediaType]; prefix != "" { - return prefix + "-" + desc.Digest.String() + return prefix + "-" + key } } switch mt := desc.MediaType; { case mt == images.MediaTypeDockerSchema2Manifest || mt == ocispec.MediaTypeImageManifest: - return "manifest-" + desc.Digest.String() + return "manifest-" + key case mt == images.MediaTypeDockerSchema2ManifestList || mt == ocispec.MediaTypeImageIndex: - return "index-" + desc.Digest.String() + return "index-" + key case images.IsLayerType(mt): - return "layer-" + desc.Digest.String() + return "layer-" + key case images.IsKnownConfig(mt): - return "config-" + desc.Digest.String() + return "config-" + key default: log.G(ctx).Warnf("reference for unknown type: %s", mt) - return "unknown-" + desc.Digest.String() + return "unknown-" + key } } @@ -115,6 +123,12 @@ func fetch(ctx context.Context, ingester content.Ingester, fetcher Fetcher, desc return err } + if desc.Size == 0 { + // most likely a poorly configured registry/web front end which responded with no + // Content-Length header; unable (not to mention useless) to commit a 0-length entry + // into the content store. Error out here otherwise the error sent back is confusing + return errors.Wrapf(errdefs.ErrInvalidArgument, "unable to fetch descriptor (%s) which reports content size of zero", desc.Digest) + } if ws.Offset == desc.Size { // If writer is already complete, commit and return err := cw.Commit(ctx, desc.Size, desc.Digest) @@ -151,7 +165,15 @@ func PushHandler(pusher Pusher, provider content.Provider) images.HandlerFunc { func push(ctx context.Context, provider content.Provider, pusher Pusher, desc ocispec.Descriptor) error { log.G(ctx).Debug("push") - cw, err := pusher.Push(ctx, desc) + var ( + cw content.Writer + err error + ) + if cs, ok := pusher.(content.Ingester); ok { + cw, err = content.OpenWriter(ctx, cs, content.WithRef(MakeRefKey(ctx, desc)), content.WithDescriptor(desc)) + } else { + cw, err = pusher.Push(ctx, desc) + } if err != nil { if !errdefs.IsAlreadyExists(err) { return err @@ -175,7 +197,8 @@ func push(ctx context.Context, provider content.Provider, pusher Pusher, desc oc // // Base handlers can be provided which will be called before any push specific // handlers. -func PushContent(ctx context.Context, pusher Pusher, desc ocispec.Descriptor, store content.Store, platform platforms.MatchComparer, wrapper func(h images.Handler) images.Handler) error { +func PushContent(ctx context.Context, pusher Pusher, desc ocispec.Descriptor, store content.Store, limiter *semaphore.Weighted, platform platforms.MatchComparer, wrapper func(h images.Handler) images.Handler) error { + var m sync.Mutex manifestStack := []ocispec.Descriptor{} @@ -207,7 +230,7 @@ func PushContent(ctx context.Context, pusher Pusher, desc ocispec.Descriptor, st handler = wrapper(handler) } - if err := images.Dispatch(ctx, handler, nil, desc); err != nil { + if err := images.Dispatch(ctx, handler, limiter, desc); err != nil { return err } diff --git a/vendor/github.com/containerd/containerd/remotes/resolver.go b/vendor/github.com/containerd/containerd/remotes/resolver.go index 914d351fd..624b14f05 100644 --- a/vendor/github.com/containerd/containerd/remotes/resolver.go +++ b/vendor/github.com/containerd/containerd/remotes/resolver.go @@ -45,6 +45,8 @@ type Resolver interface { Fetcher(ctx context.Context, ref string) (Fetcher, error) // Pusher returns a new pusher for the provided reference + // The returned Pusher should satisfy content.Ingester and concurrent attempts + // to push the same blob using the Ingester API should result in ErrUnavailable. Pusher(ctx context.Context, ref string) (Pusher, error) } diff --git a/vendor/github.com/containerd/containerd/sys/env.go b/vendor/github.com/containerd/containerd/sys/env.go deleted file mode 100644 index 8450d6275..000000000 --- a/vendor/github.com/containerd/containerd/sys/env.go +++ /dev/null @@ -1,33 +0,0 @@ -// +build !windows - -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package sys - -import "golang.org/x/sys/unix" - -// RunningPrivileged returns true if the effective user ID of the -// calling process is 0 -func RunningPrivileged() bool { - return unix.Geteuid() == 0 -} - -// RunningUnprivileged returns true if the effective user ID of the -// calling process is not 0 -func RunningUnprivileged() bool { - return !RunningPrivileged() -} diff --git a/vendor/github.com/containerd/containerd/sys/epoll.go b/vendor/github.com/containerd/containerd/sys/epoll.go deleted file mode 100644 index 28d6c2cab..000000000 --- a/vendor/github.com/containerd/containerd/sys/epoll.go +++ /dev/null @@ -1,33 +0,0 @@ -// +build linux - -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package sys - -import "golang.org/x/sys/unix" - -// EpollCreate1 is an alias for unix.EpollCreate1 -// Deprecated: use golang.org/x/sys/unix.EpollCreate1 -var EpollCreate1 = unix.EpollCreate1 - -// EpollCtl is an alias for unix.EpollCtl -// Deprecated: use golang.org/x/sys/unix.EpollCtl -var EpollCtl = unix.EpollCtl - -// EpollWait is an alias for unix.EpollWait -// Deprecated: use golang.org/x/sys/unix.EpollWait -var EpollWait = unix.EpollWait diff --git a/vendor/github.com/containerd/containerd/sys/filesys.go b/vendor/github.com/containerd/containerd/sys/filesys.go deleted file mode 100644 index 825d21d19..000000000 --- a/vendor/github.com/containerd/containerd/sys/filesys.go +++ /dev/null @@ -1,35 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package sys - -import "os" - -// IsFifo checks if a file is a (named pipe) fifo -// if the file does not exist then it returns false -func IsFifo(path string) (bool, error) { - stat, err := os.Stat(path) - if err != nil { - if os.IsNotExist(err) { - return false, nil - } - return false, err - } - if stat.Mode()&os.ModeNamedPipe == os.ModeNamedPipe { - return true, nil - } - return false, nil -} diff --git a/vendor/github.com/containerd/containerd/sys/filesys_unix.go b/vendor/github.com/containerd/containerd/sys/filesys_unix.go deleted file mode 100644 index d8329af9f..000000000 --- a/vendor/github.com/containerd/containerd/sys/filesys_unix.go +++ /dev/null @@ -1,31 +0,0 @@ -// +build !windows - -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package sys - -import "os" - -// ForceRemoveAll on unix is just a wrapper for os.RemoveAll -func ForceRemoveAll(path string) error { - return os.RemoveAll(path) -} - -// MkdirAllWithACL is a wrapper for os.MkdirAll on Unix systems. -func MkdirAllWithACL(path string, perm os.FileMode) error { - return os.MkdirAll(path, perm) -} diff --git a/vendor/github.com/containerd/containerd/sys/filesys_windows.go b/vendor/github.com/containerd/containerd/sys/filesys_windows.go deleted file mode 100644 index 2eaee2ca2..000000000 --- a/vendor/github.com/containerd/containerd/sys/filesys_windows.go +++ /dev/null @@ -1,268 +0,0 @@ -// +build windows - -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package sys - -import ( - "os" - "path/filepath" - "regexp" - "strings" - "syscall" - "unsafe" - - "github.com/Microsoft/hcsshim" - "golang.org/x/sys/windows" -) - -const ( - // SddlAdministratorsLocalSystem is local administrators plus NT AUTHORITY\System - SddlAdministratorsLocalSystem = "D:P(A;OICI;GA;;;BA)(A;OICI;GA;;;SY)" -) - -// MkdirAllWithACL is a wrapper for MkdirAll that creates a directory -// ACL'd for Builtin Administrators and Local System. -func MkdirAllWithACL(path string, perm os.FileMode) error { - return mkdirall(path, true) -} - -// MkdirAll implementation that is volume path aware for Windows. It can be used -// as a drop-in replacement for os.MkdirAll() -func MkdirAll(path string, _ os.FileMode) error { - return mkdirall(path, false) -} - -// mkdirall is a custom version of os.MkdirAll modified for use on Windows -// so that it is both volume path aware, and can create a directory with -// a DACL. -func mkdirall(path string, adminAndLocalSystem bool) error { - if re := regexp.MustCompile(`^\\\\\?\\Volume{[a-z0-9-]+}$`); re.MatchString(path) { - return nil - } - - // The rest of this method is largely copied from os.MkdirAll and should be kept - // as-is to ensure compatibility. - - // Fast path: if we can tell whether path is a directory or file, stop with success or error. - dir, err := os.Stat(path) - if err == nil { - if dir.IsDir() { - return nil - } - return &os.PathError{ - Op: "mkdir", - Path: path, - Err: syscall.ENOTDIR, - } - } - - // Slow path: make sure parent exists and then call Mkdir for path. - i := len(path) - for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator. - i-- - } - - j := i - for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element. - j-- - } - - if j > 1 { - // Create parent - err = mkdirall(path[0:j-1], adminAndLocalSystem) - if err != nil { - return err - } - } - - // Parent now exists; invoke os.Mkdir or mkdirWithACL and use its result. - if adminAndLocalSystem { - err = mkdirWithACL(path) - } else { - err = os.Mkdir(path, 0) - } - - if err != nil { - // Handle arguments like "foo/." by - // double-checking that directory doesn't exist. - dir, err1 := os.Lstat(path) - if err1 == nil && dir.IsDir() { - return nil - } - return err - } - return nil -} - -// mkdirWithACL creates a new directory. If there is an error, it will be of -// type *PathError. . -// -// This is a modified and combined version of os.Mkdir and windows.Mkdir -// in golang to cater for creating a directory am ACL permitting full -// access, with inheritance, to any subfolder/file for Built-in Administrators -// and Local System. -func mkdirWithACL(name string) error { - sa := windows.SecurityAttributes{Length: 0} - sd, err := windows.SecurityDescriptorFromString(SddlAdministratorsLocalSystem) - if err != nil { - return &os.PathError{Op: "mkdir", Path: name, Err: err} - } - sa.Length = uint32(unsafe.Sizeof(sa)) - sa.InheritHandle = 1 - sa.SecurityDescriptor = sd - - namep, err := windows.UTF16PtrFromString(name) - if err != nil { - return &os.PathError{Op: "mkdir", Path: name, Err: err} - } - - e := windows.CreateDirectory(namep, &sa) - if e != nil { - return &os.PathError{Op: "mkdir", Path: name, Err: e} - } - return nil -} - -// IsAbs is a platform-specific wrapper for filepath.IsAbs. On Windows, -// golang filepath.IsAbs does not consider a path \windows\system32 as absolute -// as it doesn't start with a drive-letter/colon combination. However, in -// docker we need to verify things such as WORKDIR /windows/system32 in -// a Dockerfile (which gets translated to \windows\system32 when being processed -// by the daemon. This SHOULD be treated as absolute from a docker processing -// perspective. -func IsAbs(path string) bool { - if !filepath.IsAbs(path) { - if !strings.HasPrefix(path, string(os.PathSeparator)) { - return false - } - } - return true -} - -// The origin of the functions below here are the golang OS and windows packages, -// slightly modified to only cope with files, not directories due to the -// specific use case. -// -// The alteration is to allow a file on Windows to be opened with -// FILE_FLAG_SEQUENTIAL_SCAN (particular for docker load), to avoid eating -// the standby list, particularly when accessing large files such as layer.tar. - -// CreateSequential creates the named file with mode 0666 (before umask), truncating -// it if it already exists. If successful, methods on the returned -// File can be used for I/O; the associated file descriptor has mode -// O_RDWR. -// If there is an error, it will be of type *PathError. -func CreateSequential(name string) (*os.File, error) { - return OpenFileSequential(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0) -} - -// OpenSequential opens the named file for reading. If successful, methods on -// the returned file can be used for reading; the associated file -// descriptor has mode O_RDONLY. -// If there is an error, it will be of type *PathError. -func OpenSequential(name string) (*os.File, error) { - return OpenFileSequential(name, os.O_RDONLY, 0) -} - -// OpenFileSequential is the generalized open call; most users will use Open -// or Create instead. -// If there is an error, it will be of type *PathError. -func OpenFileSequential(name string, flag int, _ os.FileMode) (*os.File, error) { - if name == "" { - return nil, &os.PathError{Op: "open", Path: name, Err: syscall.ENOENT} - } - r, errf := windowsOpenFileSequential(name, flag, 0) - if errf == nil { - return r, nil - } - return nil, &os.PathError{Op: "open", Path: name, Err: errf} -} - -func windowsOpenFileSequential(name string, flag int, _ os.FileMode) (file *os.File, err error) { - r, e := windowsOpenSequential(name, flag|windows.O_CLOEXEC, 0) - if e != nil { - return nil, e - } - return os.NewFile(uintptr(r), name), nil -} - -func makeInheritSa() *windows.SecurityAttributes { - var sa windows.SecurityAttributes - sa.Length = uint32(unsafe.Sizeof(sa)) - sa.InheritHandle = 1 - return &sa -} - -func windowsOpenSequential(path string, mode int, _ uint32) (fd windows.Handle, err error) { - if len(path) == 0 { - return windows.InvalidHandle, windows.ERROR_FILE_NOT_FOUND - } - pathp, err := windows.UTF16PtrFromString(path) - if err != nil { - return windows.InvalidHandle, err - } - var access uint32 - switch mode & (windows.O_RDONLY | windows.O_WRONLY | windows.O_RDWR) { - case windows.O_RDONLY: - access = windows.GENERIC_READ - case windows.O_WRONLY: - access = windows.GENERIC_WRITE - case windows.O_RDWR: - access = windows.GENERIC_READ | windows.GENERIC_WRITE - } - if mode&windows.O_CREAT != 0 { - access |= windows.GENERIC_WRITE - } - if mode&windows.O_APPEND != 0 { - access &^= windows.GENERIC_WRITE - access |= windows.FILE_APPEND_DATA - } - sharemode := uint32(windows.FILE_SHARE_READ | windows.FILE_SHARE_WRITE) - var sa *windows.SecurityAttributes - if mode&windows.O_CLOEXEC == 0 { - sa = makeInheritSa() - } - var createmode uint32 - switch { - case mode&(windows.O_CREAT|windows.O_EXCL) == (windows.O_CREAT | windows.O_EXCL): - createmode = windows.CREATE_NEW - case mode&(windows.O_CREAT|windows.O_TRUNC) == (windows.O_CREAT | windows.O_TRUNC): - createmode = windows.CREATE_ALWAYS - case mode&windows.O_CREAT == windows.O_CREAT: - createmode = windows.OPEN_ALWAYS - case mode&windows.O_TRUNC == windows.O_TRUNC: - createmode = windows.TRUNCATE_EXISTING - default: - createmode = windows.OPEN_EXISTING - } - // Use FILE_FLAG_SEQUENTIAL_SCAN rather than FILE_ATTRIBUTE_NORMAL as implemented in golang. - // https://msdn.microsoft.com/en-us/library/windows/desktop/aa363858(v=vs.85).aspx - const fileFlagSequentialScan = 0x08000000 // FILE_FLAG_SEQUENTIAL_SCAN - h, e := windows.CreateFile(pathp, access, sharemode, sa, createmode, fileFlagSequentialScan, 0) - return h, e -} - -// ForceRemoveAll is the same as os.RemoveAll, but uses hcsshim.DestroyLayer in order -// to delete container layers. -func ForceRemoveAll(path string) error { - info := hcsshim.DriverInfo{ - HomeDir: filepath.Dir(path), - } - - return hcsshim.DestroyLayer(info, filepath.Base(path)) -} diff --git a/vendor/github.com/containerd/containerd/sys/mount_linux.go b/vendor/github.com/containerd/containerd/sys/mount_linux.go deleted file mode 100644 index a21045529..000000000 --- a/vendor/github.com/containerd/containerd/sys/mount_linux.go +++ /dev/null @@ -1,145 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package sys - -import ( - "runtime" - "syscall" - "unsafe" - - "github.com/containerd/containerd/log" - "github.com/pkg/errors" - "golang.org/x/sys/unix" -) - -// FMountat performs mount from the provided directory. -func FMountat(dirfd uintptr, source, target, fstype string, flags uintptr, data string) error { - var ( - sourceP, targetP, fstypeP, dataP *byte - pid uintptr - err error - errno, status syscall.Errno - ) - - sourceP, err = syscall.BytePtrFromString(source) - if err != nil { - return err - } - - targetP, err = syscall.BytePtrFromString(target) - if err != nil { - return err - } - - fstypeP, err = syscall.BytePtrFromString(fstype) - if err != nil { - return err - } - - if data != "" { - dataP, err = syscall.BytePtrFromString(data) - if err != nil { - return err - } - } - - runtime.LockOSThread() - defer runtime.UnlockOSThread() - - var pipefds [2]int - if err := syscall.Pipe2(pipefds[:], syscall.O_CLOEXEC); err != nil { - return errors.Wrap(err, "failed to open pipe") - } - - defer func() { - // close both ends of the pipe in a deferred function, since open file - // descriptor table is shared with child - syscall.Close(pipefds[0]) - syscall.Close(pipefds[1]) - }() - - pid, errno = forkAndMountat(dirfd, - uintptr(unsafe.Pointer(sourceP)), - uintptr(unsafe.Pointer(targetP)), - uintptr(unsafe.Pointer(fstypeP)), - flags, - uintptr(unsafe.Pointer(dataP)), - pipefds[1], - ) - - if errno != 0 { - return errors.Wrap(errno, "failed to fork thread") - } - - defer func() { - _, err := unix.Wait4(int(pid), nil, 0, nil) - for err == syscall.EINTR { - _, err = unix.Wait4(int(pid), nil, 0, nil) - } - - if err != nil { - log.L.WithError(err).Debugf("failed to find pid=%d process", pid) - } - }() - - _, _, errno = syscall.RawSyscall(syscall.SYS_READ, - uintptr(pipefds[0]), - uintptr(unsafe.Pointer(&status)), - unsafe.Sizeof(status)) - if errno != 0 { - return errors.Wrap(errno, "failed to read pipe") - } - - if status != 0 { - return errors.Wrap(status, "failed to mount") - } - - return nil -} - -// forkAndMountat will fork thread, change working dir and mount. -// -// precondition: the runtime OS thread must be locked. -func forkAndMountat(dirfd uintptr, source, target, fstype, flags, data uintptr, pipefd int) (pid uintptr, errno syscall.Errno) { - - // block signal during clone - beforeFork() - - // the cloned thread shares the open file descriptor, but the thread - // never be reused by runtime. - pid, _, errno = syscall.RawSyscall6(syscall.SYS_CLONE, uintptr(syscall.SIGCHLD)|syscall.CLONE_FILES, 0, 0, 0, 0, 0) - if errno != 0 || pid != 0 { - // restore all signals - afterFork() - return - } - - // restore all signals - afterForkInChild() - - // change working dir - _, _, errno = syscall.RawSyscall(syscall.SYS_FCHDIR, dirfd, 0, 0) - if errno != 0 { - goto childerr - } - _, _, errno = syscall.RawSyscall6(syscall.SYS_MOUNT, source, target, fstype, flags, data, 0) - -childerr: - _, _, errno = syscall.RawSyscall(syscall.SYS_WRITE, uintptr(pipefd), uintptr(unsafe.Pointer(&errno)), unsafe.Sizeof(errno)) - syscall.RawSyscall(syscall.SYS_EXIT, uintptr(errno), 0, 0) - panic("unreachable") -} diff --git a/vendor/github.com/containerd/containerd/sys/oom_unix.go b/vendor/github.com/containerd/containerd/sys/oom_unix.go deleted file mode 100644 index c381e1a7e..000000000 --- a/vendor/github.com/containerd/containerd/sys/oom_unix.go +++ /dev/null @@ -1,61 +0,0 @@ -// +build !windows - -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package sys - -import ( - "fmt" - "io/ioutil" - "os" - "strconv" - "strings" -) - -const ( - // OOMScoreMaxKillable is the maximum score keeping the process killable by the oom killer - OOMScoreMaxKillable = -999 - // OOMScoreAdjMax is from OOM_SCORE_ADJ_MAX https://github.com/torvalds/linux/blob/master/include/uapi/linux/oom.h - OOMScoreAdjMax = 1000 -) - -// SetOOMScore sets the oom score for the provided pid -func SetOOMScore(pid, score int) error { - path := fmt.Sprintf("/proc/%d/oom_score_adj", pid) - f, err := os.OpenFile(path, os.O_WRONLY, 0) - if err != nil { - return err - } - defer f.Close() - if _, err = f.WriteString(strconv.Itoa(score)); err != nil { - if os.IsPermission(err) && (RunningInUserNS() || RunningUnprivileged()) { - return nil - } - return err - } - return nil -} - -// GetOOMScoreAdj gets the oom score for a process -func GetOOMScoreAdj(pid int) (int, error) { - path := fmt.Sprintf("/proc/%d/oom_score_adj", pid) - data, err := ioutil.ReadFile(path) - if err != nil { - return 0, err - } - return strconv.Atoi(strings.TrimSpace(string(data))) -} diff --git a/vendor/github.com/containerd/containerd/sys/oom_windows.go b/vendor/github.com/containerd/containerd/sys/oom_windows.go deleted file mode 100644 index 215c171f6..000000000 --- a/vendor/github.com/containerd/containerd/sys/oom_windows.go +++ /dev/null @@ -1,36 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package sys - -const ( - // OOMScoreAdjMax is not implemented on Windows - OOMScoreAdjMax = 0 -) - -// SetOOMScore sets the oom score for the process -// -// Not implemented on Windows -func SetOOMScore(pid, score int) error { - return nil -} - -// GetOOMScoreAdj gets the oom score for a process -// -// Not implemented on Windows -func GetOOMScoreAdj(pid int) (int, error) { - return 0, nil -} diff --git a/vendor/github.com/containerd/containerd/sys/socket_unix.go b/vendor/github.com/containerd/containerd/sys/socket_unix.go deleted file mode 100644 index b67cc1fa3..000000000 --- a/vendor/github.com/containerd/containerd/sys/socket_unix.go +++ /dev/null @@ -1,80 +0,0 @@ -// +build !windows - -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package sys - -import ( - "net" - "os" - "path/filepath" - - "github.com/pkg/errors" - "golang.org/x/sys/unix" -) - -// CreateUnixSocket creates a unix socket and returns the listener -func CreateUnixSocket(path string) (net.Listener, error) { - // BSDs have a 104 limit - if len(path) > 104 { - return nil, errors.Errorf("%q: unix socket path too long (> 104)", path) - } - if err := os.MkdirAll(filepath.Dir(path), 0660); err != nil { - return nil, err - } - if err := unix.Unlink(path); err != nil && !os.IsNotExist(err) { - return nil, err - } - return net.Listen("unix", path) -} - -// GetLocalListener returns a listener out of a unix socket. -func GetLocalListener(path string, uid, gid int) (net.Listener, error) { - // Ensure parent directory is created - if err := mkdirAs(filepath.Dir(path), uid, gid); err != nil { - return nil, err - } - - l, err := CreateUnixSocket(path) - if err != nil { - return l, err - } - - if err := os.Chmod(path, 0660); err != nil { - l.Close() - return nil, err - } - - if err := os.Chown(path, uid, gid); err != nil { - l.Close() - return nil, err - } - - return l, nil -} - -func mkdirAs(path string, uid, gid int) error { - if _, err := os.Stat(path); !os.IsNotExist(err) { - return err - } - - if err := os.MkdirAll(path, 0770); err != nil { - return err - } - - return os.Chown(path, uid, gid) -} diff --git a/vendor/github.com/containerd/containerd/sys/stat_bsd.go b/vendor/github.com/containerd/containerd/sys/stat_bsd.go deleted file mode 100644 index b9c95d90d..000000000 --- a/vendor/github.com/containerd/containerd/sys/stat_bsd.go +++ /dev/null @@ -1,44 +0,0 @@ -// +build darwin freebsd - -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package sys - -import ( - "syscall" - "time" -) - -// StatAtime returns the access time from a stat struct -func StatAtime(st *syscall.Stat_t) syscall.Timespec { - return st.Atimespec -} - -// StatCtime returns the created time from a stat struct -func StatCtime(st *syscall.Stat_t) syscall.Timespec { - return st.Ctimespec -} - -// StatMtime returns the modified time from a stat struct -func StatMtime(st *syscall.Stat_t) syscall.Timespec { - return st.Mtimespec -} - -// StatATimeAsTime returns the access time as a time.Time -func StatATimeAsTime(st *syscall.Stat_t) time.Time { - return time.Unix(int64(st.Atimespec.Sec), int64(st.Atimespec.Nsec)) // nolint: unconvert -} diff --git a/vendor/github.com/containerd/containerd/sys/stat_unix.go b/vendor/github.com/containerd/containerd/sys/stat_unix.go deleted file mode 100644 index 21a666dff..000000000 --- a/vendor/github.com/containerd/containerd/sys/stat_unix.go +++ /dev/null @@ -1,44 +0,0 @@ -// +build linux solaris - -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package sys - -import ( - "syscall" - "time" -) - -// StatAtime returns the Atim -func StatAtime(st *syscall.Stat_t) syscall.Timespec { - return st.Atim -} - -// StatCtime returns the Ctim -func StatCtime(st *syscall.Stat_t) syscall.Timespec { - return st.Ctim -} - -// StatMtime returns the Mtim -func StatMtime(st *syscall.Stat_t) syscall.Timespec { - return st.Mtim -} - -// StatATimeAsTime returns st.Atim as a time.Time -func StatATimeAsTime(st *syscall.Stat_t) time.Time { - return time.Unix(int64(st.Atim.Sec), int64(st.Atim.Nsec)) // nolint: unconvert -} diff --git a/vendor/github.com/containerd/containerd/sys/subprocess_unsafe_linux.go b/vendor/github.com/containerd/containerd/sys/subprocess_unsafe_linux.go deleted file mode 100644 index 6e40a9c7d..000000000 --- a/vendor/github.com/containerd/containerd/sys/subprocess_unsafe_linux.go +++ /dev/null @@ -1,30 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package sys - -import ( - _ "unsafe" // required for go:linkname. -) - -//go:linkname beforeFork syscall.runtime_BeforeFork -func beforeFork() - -//go:linkname afterFork syscall.runtime_AfterFork -func afterFork() - -//go:linkname afterForkInChild syscall.runtime_AfterForkInChild -func afterForkInChild() diff --git a/vendor/github.com/containerd/containerd/sys/subprocess_unsafe_linux.s b/vendor/github.com/containerd/containerd/sys/subprocess_unsafe_linux.s deleted file mode 100644 index c073fa4ad..000000000 --- a/vendor/github.com/containerd/containerd/sys/subprocess_unsafe_linux.s +++ /dev/null @@ -1,15 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ diff --git a/vendor/github.com/containerd/containerd/sys/userns_linux.go b/vendor/github.com/containerd/containerd/sys/userns_linux.go deleted file mode 100644 index 3cd1a2222..000000000 --- a/vendor/github.com/containerd/containerd/sys/userns_linux.go +++ /dev/null @@ -1,62 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package sys - -import ( - "bufio" - "fmt" - "os" - "sync" -) - -var ( - inUserNS bool - nsOnce sync.Once -) - -// RunningInUserNS detects whether we are currently running in a user namespace. -// Originally copied from github.com/lxc/lxd/shared/util.go -func RunningInUserNS() bool { - nsOnce.Do(func() { - file, err := os.Open("/proc/self/uid_map") - if err != nil { - // This kernel-provided file only exists if user namespaces are supported - return - } - defer file.Close() - - buf := bufio.NewReader(file) - l, _, err := buf.ReadLine() - if err != nil { - return - } - - line := string(l) - var a, b, c int64 - fmt.Sscanf(line, "%d %d %d", &a, &b, &c) - - /* - * We assume we are in the initial user namespace if we have a full - * range - 4294967295 uids starting at uid 0. - */ - if a == 0 && b == 0 && c == 4294967295 { - return - } - inUserNS = true - }) - return inUserNS -} diff --git a/vendor/github.com/containerd/containerd/version/version.go b/vendor/github.com/containerd/containerd/version/version.go index 285ee99d7..d5bacb54d 100644 --- a/vendor/github.com/containerd/containerd/version/version.go +++ b/vendor/github.com/containerd/containerd/version/version.go @@ -23,7 +23,7 @@ var ( Package = "github.com/containerd/containerd" // Version holds the complete version number. Filled in at linking time. - Version = "1.4.13+unknown" + Version = "1.5.16+unknown" // Revision is filled with the VCS (e.g. git) revision being used to build // the program at linking time. diff --git a/vendor/github.com/klauspost/compress/LICENSE b/vendor/github.com/klauspost/compress/LICENSE new file mode 100644 index 000000000..1eb75ef68 --- /dev/null +++ b/vendor/github.com/klauspost/compress/LICENSE @@ -0,0 +1,28 @@ +Copyright (c) 2012 The Go Authors. All rights reserved. +Copyright (c) 2019 Klaus Post. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/klauspost/compress/fse/README.md b/vendor/github.com/klauspost/compress/fse/README.md new file mode 100644 index 000000000..27d8ed56f --- /dev/null +++ b/vendor/github.com/klauspost/compress/fse/README.md @@ -0,0 +1,79 @@ +# Finite State Entropy + +This package provides Finite State Entropy encoding and decoding. + +Finite State Entropy (also referenced as [tANS](https://en.wikipedia.org/wiki/Asymmetric_numeral_systems#tANS)) +encoding provides a fast near-optimal symbol encoding/decoding +for byte blocks as implemented in [zstandard](https://github.com/facebook/zstd). + +This can be used for compressing input with a lot of similar input values to the smallest number of bytes. +This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders, +but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding. + +* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/fse) + +## News + + * Feb 2018: First implementation released. Consider this beta software for now. + +# Usage + +This package provides a low level interface that allows to compress single independent blocks. + +Each block is separate, and there is no built in integrity checks. +This means that the caller should keep track of block sizes and also do checksums if needed. + +Compressing a block is done via the [`Compress`](https://godoc.org/github.com/klauspost/compress/fse#Compress) function. +You must provide input and will receive the output and maybe an error. + +These error values can be returned: + +| Error | Description | +|---------------------|-----------------------------------------------------------------------------| +| `` | Everything ok, output is returned | +| `ErrIncompressible` | Returned when input is judged to be too hard to compress | +| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated | +| `(error)` | An internal error occurred. | + +As can be seen above there are errors that will be returned even under normal operation so it is important to handle these. + +To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/fse#Scratch) object +that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same +object can be used for both. + +Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this +you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output. + +Decompressing is done by calling the [`Decompress`](https://godoc.org/github.com/klauspost/compress/fse#Decompress) function. +You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back +your input was likely corrupted. + +It is important to note that a successful decoding does *not* mean your output matches your original input. +There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid. + +For more detailed usage, see examples in the [godoc documentation](https://godoc.org/github.com/klauspost/compress/fse#pkg-examples). + +# Performance + +A lot of factors are affecting speed. Block sizes and compressibility of the material are primary factors. +All compression functions are currently only running on the calling goroutine so only one core will be used per block. + +The compressor is significantly faster if symbols are kept as small as possible. The highest byte value of the input +is used to reduce some of the processing, so if all your input is above byte value 64 for instance, it may be +beneficial to transpose all your input values down by 64. + +With moderate block sizes around 64k speed are typically 200MB/s per core for compression and +around 300MB/s decompression speed. + +The same hardware typically does Huffman (deflate) encoding at 125MB/s and decompression at 100MB/s. + +# Plans + +At one point, more internals will be exposed to facilitate more "expert" usage of the components. + +A streaming interface is also likely to be implemented. Likely compatible with [FSE stream format](https://github.com/Cyan4973/FiniteStateEntropy/blob/dev/programs/fileio.c#L261). + +# Contributing + +Contributions are always welcome. Be aware that adding public functions will require good justification and breaking +changes will likely not be accepted. If in doubt open an issue before writing the PR. \ No newline at end of file diff --git a/vendor/github.com/klauspost/compress/fse/bitreader.go b/vendor/github.com/klauspost/compress/fse/bitreader.go new file mode 100644 index 000000000..b9db204f5 --- /dev/null +++ b/vendor/github.com/klauspost/compress/fse/bitreader.go @@ -0,0 +1,107 @@ +// Copyright 2018 Klaus Post. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// Based on work Copyright (c) 2013, Yann Collet, released under BSD License. + +package fse + +import ( + "errors" + "io" +) + +// bitReader reads a bitstream in reverse. +// The last set bit indicates the start of the stream and is used +// for aligning the input. +type bitReader struct { + in []byte + off uint // next byte to read is at in[off - 1] + value uint64 + bitsRead uint8 +} + +// init initializes and resets the bit reader. +func (b *bitReader) init(in []byte) error { + if len(in) < 1 { + return errors.New("corrupt stream: too short") + } + b.in = in + b.off = uint(len(in)) + // The highest bit of the last byte indicates where to start + v := in[len(in)-1] + if v == 0 { + return errors.New("corrupt stream, did not find end of stream") + } + b.bitsRead = 64 + b.value = 0 + b.fill() + b.fill() + b.bitsRead += 8 - uint8(highBits(uint32(v))) + return nil +} + +// getBits will return n bits. n can be 0. +func (b *bitReader) getBits(n uint8) uint16 { + if n == 0 || b.bitsRead >= 64 { + return 0 + } + return b.getBitsFast(n) +} + +// getBitsFast requires that at least one bit is requested every time. +// There are no checks if the buffer is filled. +func (b *bitReader) getBitsFast(n uint8) uint16 { + const regMask = 64 - 1 + v := uint16((b.value << (b.bitsRead & regMask)) >> ((regMask + 1 - n) & regMask)) + b.bitsRead += n + return v +} + +// fillFast() will make sure at least 32 bits are available. +// There must be at least 4 bytes available. +func (b *bitReader) fillFast() { + if b.bitsRead < 32 { + return + } + // Do single re-slice to avoid bounds checks. + v := b.in[b.off-4 : b.off] + low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) + b.value = (b.value << 32) | uint64(low) + b.bitsRead -= 32 + b.off -= 4 +} + +// fill() will make sure at least 32 bits are available. +func (b *bitReader) fill() { + if b.bitsRead < 32 { + return + } + if b.off > 4 { + v := b.in[b.off-4 : b.off] + low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) + b.value = (b.value << 32) | uint64(low) + b.bitsRead -= 32 + b.off -= 4 + return + } + for b.off > 0 { + b.value = (b.value << 8) | uint64(b.in[b.off-1]) + b.bitsRead -= 8 + b.off-- + } +} + +// finished returns true if all bits have been read from the bit stream. +func (b *bitReader) finished() bool { + return b.off == 0 && b.bitsRead >= 64 +} + +// close the bitstream and returns an error if out-of-buffer reads occurred. +func (b *bitReader) close() error { + // Release reference. + b.in = nil + if b.bitsRead > 64 { + return io.ErrUnexpectedEOF + } + return nil +} diff --git a/vendor/github.com/klauspost/compress/fse/bitwriter.go b/vendor/github.com/klauspost/compress/fse/bitwriter.go new file mode 100644 index 000000000..43e463611 --- /dev/null +++ b/vendor/github.com/klauspost/compress/fse/bitwriter.go @@ -0,0 +1,168 @@ +// Copyright 2018 Klaus Post. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// Based on work Copyright (c) 2013, Yann Collet, released under BSD License. + +package fse + +import "fmt" + +// bitWriter will write bits. +// First bit will be LSB of the first byte of output. +type bitWriter struct { + bitContainer uint64 + nBits uint8 + out []byte +} + +// bitMask16 is bitmasks. Has extra to avoid bounds check. +var bitMask16 = [32]uint16{ + 0, 1, 3, 7, 0xF, 0x1F, + 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, + 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0xFFFF, + 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, + 0xFFFF, 0xFFFF} /* up to 16 bits */ + +// addBits16NC will add up to 16 bits. +// It will not check if there is space for them, +// so the caller must ensure that it has flushed recently. +func (b *bitWriter) addBits16NC(value uint16, bits uint8) { + b.bitContainer |= uint64(value&bitMask16[bits&31]) << (b.nBits & 63) + b.nBits += bits +} + +// addBits16Clean will add up to 16 bits. value may not contain more set bits than indicated. +// It will not check if there is space for them, so the caller must ensure that it has flushed recently. +func (b *bitWriter) addBits16Clean(value uint16, bits uint8) { + b.bitContainer |= uint64(value) << (b.nBits & 63) + b.nBits += bits +} + +// addBits16ZeroNC will add up to 16 bits. +// It will not check if there is space for them, +// so the caller must ensure that it has flushed recently. +// This is fastest if bits can be zero. +func (b *bitWriter) addBits16ZeroNC(value uint16, bits uint8) { + if bits == 0 { + return + } + value <<= (16 - bits) & 15 + value >>= (16 - bits) & 15 + b.bitContainer |= uint64(value) << (b.nBits & 63) + b.nBits += bits +} + +// flush will flush all pending full bytes. +// There will be at least 56 bits available for writing when this has been called. +// Using flush32 is faster, but leaves less space for writing. +func (b *bitWriter) flush() { + v := b.nBits >> 3 + switch v { + case 0: + case 1: + b.out = append(b.out, + byte(b.bitContainer), + ) + case 2: + b.out = append(b.out, + byte(b.bitContainer), + byte(b.bitContainer>>8), + ) + case 3: + b.out = append(b.out, + byte(b.bitContainer), + byte(b.bitContainer>>8), + byte(b.bitContainer>>16), + ) + case 4: + b.out = append(b.out, + byte(b.bitContainer), + byte(b.bitContainer>>8), + byte(b.bitContainer>>16), + byte(b.bitContainer>>24), + ) + case 5: + b.out = append(b.out, + byte(b.bitContainer), + byte(b.bitContainer>>8), + byte(b.bitContainer>>16), + byte(b.bitContainer>>24), + byte(b.bitContainer>>32), + ) + case 6: + b.out = append(b.out, + byte(b.bitContainer), + byte(b.bitContainer>>8), + byte(b.bitContainer>>16), + byte(b.bitContainer>>24), + byte(b.bitContainer>>32), + byte(b.bitContainer>>40), + ) + case 7: + b.out = append(b.out, + byte(b.bitContainer), + byte(b.bitContainer>>8), + byte(b.bitContainer>>16), + byte(b.bitContainer>>24), + byte(b.bitContainer>>32), + byte(b.bitContainer>>40), + byte(b.bitContainer>>48), + ) + case 8: + b.out = append(b.out, + byte(b.bitContainer), + byte(b.bitContainer>>8), + byte(b.bitContainer>>16), + byte(b.bitContainer>>24), + byte(b.bitContainer>>32), + byte(b.bitContainer>>40), + byte(b.bitContainer>>48), + byte(b.bitContainer>>56), + ) + default: + panic(fmt.Errorf("bits (%d) > 64", b.nBits)) + } + b.bitContainer >>= v << 3 + b.nBits &= 7 +} + +// flush32 will flush out, so there are at least 32 bits available for writing. +func (b *bitWriter) flush32() { + if b.nBits < 32 { + return + } + b.out = append(b.out, + byte(b.bitContainer), + byte(b.bitContainer>>8), + byte(b.bitContainer>>16), + byte(b.bitContainer>>24)) + b.nBits -= 32 + b.bitContainer >>= 32 +} + +// flushAlign will flush remaining full bytes and align to next byte boundary. +func (b *bitWriter) flushAlign() { + nbBytes := (b.nBits + 7) >> 3 + for i := uint8(0); i < nbBytes; i++ { + b.out = append(b.out, byte(b.bitContainer>>(i*8))) + } + b.nBits = 0 + b.bitContainer = 0 +} + +// close will write the alignment bit and write the final byte(s) +// to the output. +func (b *bitWriter) close() error { + // End mark + b.addBits16Clean(1, 1) + // flush until next byte. + b.flushAlign() + return nil +} + +// reset and continue writing by appending to out. +func (b *bitWriter) reset(out []byte) { + b.bitContainer = 0 + b.nBits = 0 + b.out = out +} diff --git a/vendor/github.com/klauspost/compress/fse/bytereader.go b/vendor/github.com/klauspost/compress/fse/bytereader.go new file mode 100644 index 000000000..f228a46cd --- /dev/null +++ b/vendor/github.com/klauspost/compress/fse/bytereader.go @@ -0,0 +1,56 @@ +// Copyright 2018 Klaus Post. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// Based on work Copyright (c) 2013, Yann Collet, released under BSD License. + +package fse + +// byteReader provides a byte reader that reads +// little endian values from a byte stream. +// The input stream is manually advanced. +// The reader performs no bounds checks. +type byteReader struct { + b []byte + off int +} + +// init will initialize the reader and set the input. +func (b *byteReader) init(in []byte) { + b.b = in + b.off = 0 +} + +// advance the stream b n bytes. +func (b *byteReader) advance(n uint) { + b.off += int(n) +} + +// Int32 returns a little endian int32 starting at current offset. +func (b byteReader) Int32() int32 { + b2 := b.b[b.off : b.off+4 : b.off+4] + v3 := int32(b2[3]) + v2 := int32(b2[2]) + v1 := int32(b2[1]) + v0 := int32(b2[0]) + return v0 | (v1 << 8) | (v2 << 16) | (v3 << 24) +} + +// Uint32 returns a little endian uint32 starting at current offset. +func (b byteReader) Uint32() uint32 { + b2 := b.b[b.off : b.off+4 : b.off+4] + v3 := uint32(b2[3]) + v2 := uint32(b2[2]) + v1 := uint32(b2[1]) + v0 := uint32(b2[0]) + return v0 | (v1 << 8) | (v2 << 16) | (v3 << 24) +} + +// unread returns the unread portion of the input. +func (b byteReader) unread() []byte { + return b.b[b.off:] +} + +// remain will return the number of bytes remaining. +func (b byteReader) remain() int { + return len(b.b) - b.off +} diff --git a/vendor/github.com/klauspost/compress/fse/compress.go b/vendor/github.com/klauspost/compress/fse/compress.go new file mode 100644 index 000000000..b69237c9b --- /dev/null +++ b/vendor/github.com/klauspost/compress/fse/compress.go @@ -0,0 +1,684 @@ +// Copyright 2018 Klaus Post. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// Based on work Copyright (c) 2013, Yann Collet, released under BSD License. + +package fse + +import ( + "errors" + "fmt" +) + +// Compress the input bytes. Input must be < 2GB. +// Provide a Scratch buffer to avoid memory allocations. +// Note that the output is also kept in the scratch buffer. +// If input is too hard to compress, ErrIncompressible is returned. +// If input is a single byte value repeated ErrUseRLE is returned. +func Compress(in []byte, s *Scratch) ([]byte, error) { + if len(in) <= 1 { + return nil, ErrIncompressible + } + if len(in) > (2<<30)-1 { + return nil, errors.New("input too big, must be < 2GB") + } + s, err := s.prepare(in) + if err != nil { + return nil, err + } + + // Create histogram, if none was provided. + maxCount := s.maxCount + if maxCount == 0 { + maxCount = s.countSimple(in) + } + // Reset for next run. + s.clearCount = true + s.maxCount = 0 + if maxCount == len(in) { + // One symbol, use RLE + return nil, ErrUseRLE + } + if maxCount == 1 || maxCount < (len(in)>>7) { + // Each symbol present maximum once or too well distributed. + return nil, ErrIncompressible + } + s.optimalTableLog() + err = s.normalizeCount() + if err != nil { + return nil, err + } + err = s.writeCount() + if err != nil { + return nil, err + } + + if false { + err = s.validateNorm() + if err != nil { + return nil, err + } + } + + err = s.buildCTable() + if err != nil { + return nil, err + } + err = s.compress(in) + if err != nil { + return nil, err + } + s.Out = s.bw.out + // Check if we compressed. + if len(s.Out) >= len(in) { + return nil, ErrIncompressible + } + return s.Out, nil +} + +// cState contains the compression state of a stream. +type cState struct { + bw *bitWriter + stateTable []uint16 + state uint16 +} + +// init will initialize the compression state to the first symbol of the stream. +func (c *cState) init(bw *bitWriter, ct *cTable, tableLog uint8, first symbolTransform) { + c.bw = bw + c.stateTable = ct.stateTable + + nbBitsOut := (first.deltaNbBits + (1 << 15)) >> 16 + im := int32((nbBitsOut << 16) - first.deltaNbBits) + lu := (im >> nbBitsOut) + first.deltaFindState + c.state = c.stateTable[lu] + return +} + +// encode the output symbol provided and write it to the bitstream. +func (c *cState) encode(symbolTT symbolTransform) { + nbBitsOut := (uint32(c.state) + symbolTT.deltaNbBits) >> 16 + dstState := int32(c.state>>(nbBitsOut&15)) + symbolTT.deltaFindState + c.bw.addBits16NC(c.state, uint8(nbBitsOut)) + c.state = c.stateTable[dstState] +} + +// encode the output symbol provided and write it to the bitstream. +func (c *cState) encodeZero(symbolTT symbolTransform) { + nbBitsOut := (uint32(c.state) + symbolTT.deltaNbBits) >> 16 + dstState := int32(c.state>>(nbBitsOut&15)) + symbolTT.deltaFindState + c.bw.addBits16ZeroNC(c.state, uint8(nbBitsOut)) + c.state = c.stateTable[dstState] +} + +// flush will write the tablelog to the output and flush the remaining full bytes. +func (c *cState) flush(tableLog uint8) { + c.bw.flush32() + c.bw.addBits16NC(c.state, tableLog) + c.bw.flush() +} + +// compress is the main compression loop that will encode the input from the last byte to the first. +func (s *Scratch) compress(src []byte) error { + if len(src) <= 2 { + return errors.New("compress: src too small") + } + tt := s.ct.symbolTT[:256] + s.bw.reset(s.Out) + + // Our two states each encodes every second byte. + // Last byte encoded (first byte decoded) will always be encoded by c1. + var c1, c2 cState + + // Encode so remaining size is divisible by 4. + ip := len(src) + if ip&1 == 1 { + c1.init(&s.bw, &s.ct, s.actualTableLog, tt[src[ip-1]]) + c2.init(&s.bw, &s.ct, s.actualTableLog, tt[src[ip-2]]) + c1.encodeZero(tt[src[ip-3]]) + ip -= 3 + } else { + c2.init(&s.bw, &s.ct, s.actualTableLog, tt[src[ip-1]]) + c1.init(&s.bw, &s.ct, s.actualTableLog, tt[src[ip-2]]) + ip -= 2 + } + if ip&2 != 0 { + c2.encodeZero(tt[src[ip-1]]) + c1.encodeZero(tt[src[ip-2]]) + ip -= 2 + } + + // Main compression loop. + switch { + case !s.zeroBits && s.actualTableLog <= 8: + // We can encode 4 symbols without requiring a flush. + // We do not need to check if any output is 0 bits. + for ip >= 4 { + s.bw.flush32() + v3, v2, v1, v0 := src[ip-4], src[ip-3], src[ip-2], src[ip-1] + c2.encode(tt[v0]) + c1.encode(tt[v1]) + c2.encode(tt[v2]) + c1.encode(tt[v3]) + ip -= 4 + } + case !s.zeroBits: + // We do not need to check if any output is 0 bits. + for ip >= 4 { + s.bw.flush32() + v3, v2, v1, v0 := src[ip-4], src[ip-3], src[ip-2], src[ip-1] + c2.encode(tt[v0]) + c1.encode(tt[v1]) + s.bw.flush32() + c2.encode(tt[v2]) + c1.encode(tt[v3]) + ip -= 4 + } + case s.actualTableLog <= 8: + // We can encode 4 symbols without requiring a flush + for ip >= 4 { + s.bw.flush32() + v3, v2, v1, v0 := src[ip-4], src[ip-3], src[ip-2], src[ip-1] + c2.encodeZero(tt[v0]) + c1.encodeZero(tt[v1]) + c2.encodeZero(tt[v2]) + c1.encodeZero(tt[v3]) + ip -= 4 + } + default: + for ip >= 4 { + s.bw.flush32() + v3, v2, v1, v0 := src[ip-4], src[ip-3], src[ip-2], src[ip-1] + c2.encodeZero(tt[v0]) + c1.encodeZero(tt[v1]) + s.bw.flush32() + c2.encodeZero(tt[v2]) + c1.encodeZero(tt[v3]) + ip -= 4 + } + } + + // Flush final state. + // Used to initialize state when decoding. + c2.flush(s.actualTableLog) + c1.flush(s.actualTableLog) + + return s.bw.close() +} + +// writeCount will write the normalized histogram count to header. +// This is read back by readNCount. +func (s *Scratch) writeCount() error { + var ( + tableLog = s.actualTableLog + tableSize = 1 << tableLog + previous0 bool + charnum uint16 + + maxHeaderSize = ((int(s.symbolLen) * int(tableLog)) >> 3) + 3 + + // Write Table Size + bitStream = uint32(tableLog - minTablelog) + bitCount = uint(4) + remaining = int16(tableSize + 1) /* +1 for extra accuracy */ + threshold = int16(tableSize) + nbBits = uint(tableLog + 1) + ) + if cap(s.Out) < maxHeaderSize { + s.Out = make([]byte, 0, s.br.remain()+maxHeaderSize) + } + outP := uint(0) + out := s.Out[:maxHeaderSize] + + // stops at 1 + for remaining > 1 { + if previous0 { + start := charnum + for s.norm[charnum] == 0 { + charnum++ + } + for charnum >= start+24 { + start += 24 + bitStream += uint32(0xFFFF) << bitCount + out[outP] = byte(bitStream) + out[outP+1] = byte(bitStream >> 8) + outP += 2 + bitStream >>= 16 + } + for charnum >= start+3 { + start += 3 + bitStream += 3 << bitCount + bitCount += 2 + } + bitStream += uint32(charnum-start) << bitCount + bitCount += 2 + if bitCount > 16 { + out[outP] = byte(bitStream) + out[outP+1] = byte(bitStream >> 8) + outP += 2 + bitStream >>= 16 + bitCount -= 16 + } + } + + count := s.norm[charnum] + charnum++ + max := (2*threshold - 1) - remaining + if count < 0 { + remaining += count + } else { + remaining -= count + } + count++ // +1 for extra accuracy + if count >= threshold { + count += max // [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ + } + bitStream += uint32(count) << bitCount + bitCount += nbBits + if count < max { + bitCount-- + } + + previous0 = count == 1 + if remaining < 1 { + return errors.New("internal error: remaining<1") + } + for remaining < threshold { + nbBits-- + threshold >>= 1 + } + + if bitCount > 16 { + out[outP] = byte(bitStream) + out[outP+1] = byte(bitStream >> 8) + outP += 2 + bitStream >>= 16 + bitCount -= 16 + } + } + + out[outP] = byte(bitStream) + out[outP+1] = byte(bitStream >> 8) + outP += (bitCount + 7) / 8 + + if uint16(charnum) > s.symbolLen { + return errors.New("internal error: charnum > s.symbolLen") + } + s.Out = out[:outP] + return nil +} + +// symbolTransform contains the state transform for a symbol. +type symbolTransform struct { + deltaFindState int32 + deltaNbBits uint32 +} + +// String prints values as a human readable string. +func (s symbolTransform) String() string { + return fmt.Sprintf("dnbits: %08x, fs:%d", s.deltaNbBits, s.deltaFindState) +} + +// cTable contains tables used for compression. +type cTable struct { + tableSymbol []byte + stateTable []uint16 + symbolTT []symbolTransform +} + +// allocCtable will allocate tables needed for compression. +// If existing tables a re big enough, they are simply re-used. +func (s *Scratch) allocCtable() { + tableSize := 1 << s.actualTableLog + // get tableSymbol that is big enough. + if cap(s.ct.tableSymbol) < int(tableSize) { + s.ct.tableSymbol = make([]byte, tableSize) + } + s.ct.tableSymbol = s.ct.tableSymbol[:tableSize] + + ctSize := tableSize + if cap(s.ct.stateTable) < ctSize { + s.ct.stateTable = make([]uint16, ctSize) + } + s.ct.stateTable = s.ct.stateTable[:ctSize] + + if cap(s.ct.symbolTT) < 256 { + s.ct.symbolTT = make([]symbolTransform, 256) + } + s.ct.symbolTT = s.ct.symbolTT[:256] +} + +// buildCTable will populate the compression table so it is ready to be used. +func (s *Scratch) buildCTable() error { + tableSize := uint32(1 << s.actualTableLog) + highThreshold := tableSize - 1 + var cumul [maxSymbolValue + 2]int16 + + s.allocCtable() + tableSymbol := s.ct.tableSymbol[:tableSize] + // symbol start positions + { + cumul[0] = 0 + for ui, v := range s.norm[:s.symbolLen-1] { + u := byte(ui) // one less than reference + if v == -1 { + // Low proba symbol + cumul[u+1] = cumul[u] + 1 + tableSymbol[highThreshold] = u + highThreshold-- + } else { + cumul[u+1] = cumul[u] + v + } + } + // Encode last symbol separately to avoid overflowing u + u := int(s.symbolLen - 1) + v := s.norm[s.symbolLen-1] + if v == -1 { + // Low proba symbol + cumul[u+1] = cumul[u] + 1 + tableSymbol[highThreshold] = byte(u) + highThreshold-- + } else { + cumul[u+1] = cumul[u] + v + } + if uint32(cumul[s.symbolLen]) != tableSize { + return fmt.Errorf("internal error: expected cumul[s.symbolLen] (%d) == tableSize (%d)", cumul[s.symbolLen], tableSize) + } + cumul[s.symbolLen] = int16(tableSize) + 1 + } + // Spread symbols + s.zeroBits = false + { + step := tableStep(tableSize) + tableMask := tableSize - 1 + var position uint32 + // if any symbol > largeLimit, we may have 0 bits output. + largeLimit := int16(1 << (s.actualTableLog - 1)) + for ui, v := range s.norm[:s.symbolLen] { + symbol := byte(ui) + if v > largeLimit { + s.zeroBits = true + } + for nbOccurrences := int16(0); nbOccurrences < v; nbOccurrences++ { + tableSymbol[position] = symbol + position = (position + step) & tableMask + for position > highThreshold { + position = (position + step) & tableMask + } /* Low proba area */ + } + } + + // Check if we have gone through all positions + if position != 0 { + return errors.New("position!=0") + } + } + + // Build table + table := s.ct.stateTable + { + tsi := int(tableSize) + for u, v := range tableSymbol { + // TableU16 : sorted by symbol order; gives next state value + table[cumul[v]] = uint16(tsi + u) + cumul[v]++ + } + } + + // Build Symbol Transformation Table + { + total := int16(0) + symbolTT := s.ct.symbolTT[:s.symbolLen] + tableLog := s.actualTableLog + tl := (uint32(tableLog) << 16) - (1 << tableLog) + for i, v := range s.norm[:s.symbolLen] { + switch v { + case 0: + case -1, 1: + symbolTT[i].deltaNbBits = tl + symbolTT[i].deltaFindState = int32(total - 1) + total++ + default: + maxBitsOut := uint32(tableLog) - highBits(uint32(v-1)) + minStatePlus := uint32(v) << maxBitsOut + symbolTT[i].deltaNbBits = (maxBitsOut << 16) - minStatePlus + symbolTT[i].deltaFindState = int32(total - v) + total += v + } + } + if total != int16(tableSize) { + return fmt.Errorf("total mismatch %d (got) != %d (want)", total, tableSize) + } + } + return nil +} + +// countSimple will create a simple histogram in s.count. +// Returns the biggest count. +// Does not update s.clearCount. +func (s *Scratch) countSimple(in []byte) (max int) { + for _, v := range in { + s.count[v]++ + } + m := uint32(0) + for i, v := range s.count[:] { + if v > m { + m = v + } + if v > 0 { + s.symbolLen = uint16(i) + 1 + } + } + return int(m) +} + +// minTableLog provides the minimum logSize to safely represent a distribution. +func (s *Scratch) minTableLog() uint8 { + minBitsSrc := highBits(uint32(s.br.remain()-1)) + 1 + minBitsSymbols := highBits(uint32(s.symbolLen-1)) + 2 + if minBitsSrc < minBitsSymbols { + return uint8(minBitsSrc) + } + return uint8(minBitsSymbols) +} + +// optimalTableLog calculates and sets the optimal tableLog in s.actualTableLog +func (s *Scratch) optimalTableLog() { + tableLog := s.TableLog + minBits := s.minTableLog() + maxBitsSrc := uint8(highBits(uint32(s.br.remain()-1))) - 2 + if maxBitsSrc < tableLog { + // Accuracy can be reduced + tableLog = maxBitsSrc + } + if minBits > tableLog { + tableLog = minBits + } + // Need a minimum to safely represent all symbol values + if tableLog < minTablelog { + tableLog = minTablelog + } + if tableLog > maxTableLog { + tableLog = maxTableLog + } + s.actualTableLog = tableLog +} + +var rtbTable = [...]uint32{0, 473195, 504333, 520860, 550000, 700000, 750000, 830000} + +// normalizeCount will normalize the count of the symbols so +// the total is equal to the table size. +func (s *Scratch) normalizeCount() error { + var ( + tableLog = s.actualTableLog + scale = 62 - uint64(tableLog) + step = (1 << 62) / uint64(s.br.remain()) + vStep = uint64(1) << (scale - 20) + stillToDistribute = int16(1 << tableLog) + largest int + largestP int16 + lowThreshold = (uint32)(s.br.remain() >> tableLog) + ) + + for i, cnt := range s.count[:s.symbolLen] { + // already handled + // if (count[s] == s.length) return 0; /* rle special case */ + + if cnt == 0 { + s.norm[i] = 0 + continue + } + if cnt <= lowThreshold { + s.norm[i] = -1 + stillToDistribute-- + } else { + proba := (int16)((uint64(cnt) * step) >> scale) + if proba < 8 { + restToBeat := vStep * uint64(rtbTable[proba]) + v := uint64(cnt)*step - (uint64(proba) << scale) + if v > restToBeat { + proba++ + } + } + if proba > largestP { + largestP = proba + largest = i + } + s.norm[i] = proba + stillToDistribute -= proba + } + } + + if -stillToDistribute >= (s.norm[largest] >> 1) { + // corner case, need another normalization method + return s.normalizeCount2() + } + s.norm[largest] += stillToDistribute + return nil +} + +// Secondary normalization method. +// To be used when primary method fails. +func (s *Scratch) normalizeCount2() error { + const notYetAssigned = -2 + var ( + distributed uint32 + total = uint32(s.br.remain()) + tableLog = s.actualTableLog + lowThreshold = uint32(total >> tableLog) + lowOne = uint32((total * 3) >> (tableLog + 1)) + ) + for i, cnt := range s.count[:s.symbolLen] { + if cnt == 0 { + s.norm[i] = 0 + continue + } + if cnt <= lowThreshold { + s.norm[i] = -1 + distributed++ + total -= cnt + continue + } + if cnt <= lowOne { + s.norm[i] = 1 + distributed++ + total -= cnt + continue + } + s.norm[i] = notYetAssigned + } + toDistribute := (1 << tableLog) - distributed + + if (total / toDistribute) > lowOne { + // risk of rounding to zero + lowOne = uint32((total * 3) / (toDistribute * 2)) + for i, cnt := range s.count[:s.symbolLen] { + if (s.norm[i] == notYetAssigned) && (cnt <= lowOne) { + s.norm[i] = 1 + distributed++ + total -= cnt + continue + } + } + toDistribute = (1 << tableLog) - distributed + } + if distributed == uint32(s.symbolLen)+1 { + // all values are pretty poor; + // probably incompressible data (should have already been detected); + // find max, then give all remaining points to max + var maxV int + var maxC uint32 + for i, cnt := range s.count[:s.symbolLen] { + if cnt > maxC { + maxV = i + maxC = cnt + } + } + s.norm[maxV] += int16(toDistribute) + return nil + } + + if total == 0 { + // all of the symbols were low enough for the lowOne or lowThreshold + for i := uint32(0); toDistribute > 0; i = (i + 1) % (uint32(s.symbolLen)) { + if s.norm[i] > 0 { + toDistribute-- + s.norm[i]++ + } + } + return nil + } + + var ( + vStepLog = 62 - uint64(tableLog) + mid = uint64((1 << (vStepLog - 1)) - 1) + rStep = (((1 << vStepLog) * uint64(toDistribute)) + mid) / uint64(total) // scale on remaining + tmpTotal = mid + ) + for i, cnt := range s.count[:s.symbolLen] { + if s.norm[i] == notYetAssigned { + var ( + end = tmpTotal + uint64(cnt)*rStep + sStart = uint32(tmpTotal >> vStepLog) + sEnd = uint32(end >> vStepLog) + weight = sEnd - sStart + ) + if weight < 1 { + return errors.New("weight < 1") + } + s.norm[i] = int16(weight) + tmpTotal = end + } + } + return nil +} + +// validateNorm validates the normalized histogram table. +func (s *Scratch) validateNorm() (err error) { + var total int + for _, v := range s.norm[:s.symbolLen] { + if v >= 0 { + total += int(v) + } else { + total -= int(v) + } + } + defer func() { + if err == nil { + return + } + fmt.Printf("selected TableLog: %d, Symbol length: %d\n", s.actualTableLog, s.symbolLen) + for i, v := range s.norm[:s.symbolLen] { + fmt.Printf("%3d: %5d -> %4d \n", i, s.count[i], v) + } + }() + if total != (1 << s.actualTableLog) { + return fmt.Errorf("warning: Total == %d != %d", total, 1< tablelogAbsoluteMax { + return errors.New("tableLog too large") + } + bitStream >>= 4 + bitCount := uint(4) + + s.actualTableLog = uint8(nbBits) + remaining := int32((1 << nbBits) + 1) + threshold := int32(1 << nbBits) + gotTotal := int32(0) + nbBits++ + + for remaining > 1 { + if previous0 { + n0 := charnum + for (bitStream & 0xFFFF) == 0xFFFF { + n0 += 24 + if b.off < iend-5 { + b.advance(2) + bitStream = b.Uint32() >> bitCount + } else { + bitStream >>= 16 + bitCount += 16 + } + } + for (bitStream & 3) == 3 { + n0 += 3 + bitStream >>= 2 + bitCount += 2 + } + n0 += uint16(bitStream & 3) + bitCount += 2 + if n0 > maxSymbolValue { + return errors.New("maxSymbolValue too small") + } + for charnum < n0 { + s.norm[charnum&0xff] = 0 + charnum++ + } + + if b.off <= iend-7 || b.off+int(bitCount>>3) <= iend-4 { + b.advance(bitCount >> 3) + bitCount &= 7 + bitStream = b.Uint32() >> bitCount + } else { + bitStream >>= 2 + } + } + + max := (2*(threshold) - 1) - (remaining) + var count int32 + + if (int32(bitStream) & (threshold - 1)) < max { + count = int32(bitStream) & (threshold - 1) + bitCount += nbBits - 1 + } else { + count = int32(bitStream) & (2*threshold - 1) + if count >= threshold { + count -= max + } + bitCount += nbBits + } + + count-- // extra accuracy + if count < 0 { + // -1 means +1 + remaining += count + gotTotal -= count + } else { + remaining -= count + gotTotal += count + } + s.norm[charnum&0xff] = int16(count) + charnum++ + previous0 = count == 0 + for remaining < threshold { + nbBits-- + threshold >>= 1 + } + if b.off <= iend-7 || b.off+int(bitCount>>3) <= iend-4 { + b.advance(bitCount >> 3) + bitCount &= 7 + } else { + bitCount -= (uint)(8 * (len(b.b) - 4 - b.off)) + b.off = len(b.b) - 4 + } + bitStream = b.Uint32() >> (bitCount & 31) + } + s.symbolLen = charnum + + if s.symbolLen <= 1 { + return fmt.Errorf("symbolLen (%d) too small", s.symbolLen) + } + if s.symbolLen > maxSymbolValue+1 { + return fmt.Errorf("symbolLen (%d) too big", s.symbolLen) + } + if remaining != 1 { + return fmt.Errorf("corruption detected (remaining %d != 1)", remaining) + } + if bitCount > 32 { + return fmt.Errorf("corruption detected (bitCount %d > 32)", bitCount) + } + if gotTotal != 1<> 3) + return nil +} + +// decSymbol contains information about a state entry, +// Including the state offset base, the output symbol and +// the number of bits to read for the low part of the destination state. +type decSymbol struct { + newState uint16 + symbol uint8 + nbBits uint8 +} + +// allocDtable will allocate decoding tables if they are not big enough. +func (s *Scratch) allocDtable() { + tableSize := 1 << s.actualTableLog + if cap(s.decTable) < int(tableSize) { + s.decTable = make([]decSymbol, tableSize) + } + s.decTable = s.decTable[:tableSize] + + if cap(s.ct.tableSymbol) < 256 { + s.ct.tableSymbol = make([]byte, 256) + } + s.ct.tableSymbol = s.ct.tableSymbol[:256] + + if cap(s.ct.stateTable) < 256 { + s.ct.stateTable = make([]uint16, 256) + } + s.ct.stateTable = s.ct.stateTable[:256] +} + +// buildDtable will build the decoding table. +func (s *Scratch) buildDtable() error { + tableSize := uint32(1 << s.actualTableLog) + highThreshold := tableSize - 1 + s.allocDtable() + symbolNext := s.ct.stateTable[:256] + + // Init, lay down lowprob symbols + s.zeroBits = false + { + largeLimit := int16(1 << (s.actualTableLog - 1)) + for i, v := range s.norm[:s.symbolLen] { + if v == -1 { + s.decTable[highThreshold].symbol = uint8(i) + highThreshold-- + symbolNext[i] = 1 + } else { + if v >= largeLimit { + s.zeroBits = true + } + symbolNext[i] = uint16(v) + } + } + } + // Spread symbols + { + tableMask := tableSize - 1 + step := tableStep(tableSize) + position := uint32(0) + for ss, v := range s.norm[:s.symbolLen] { + for i := 0; i < int(v); i++ { + s.decTable[position].symbol = uint8(ss) + position = (position + step) & tableMask + for position > highThreshold { + // lowprob area + position = (position + step) & tableMask + } + } + } + if position != 0 { + // position must reach all cells once, otherwise normalizedCounter is incorrect + return errors.New("corrupted input (position != 0)") + } + } + + // Build Decoding table + { + tableSize := uint16(1 << s.actualTableLog) + for u, v := range s.decTable { + symbol := v.symbol + nextState := symbolNext[symbol] + symbolNext[symbol] = nextState + 1 + nBits := s.actualTableLog - byte(highBits(uint32(nextState))) + s.decTable[u].nbBits = nBits + newState := (nextState << nBits) - tableSize + if newState >= tableSize { + return fmt.Errorf("newState (%d) outside table size (%d)", newState, tableSize) + } + if newState == uint16(u) && nBits == 0 { + // Seems weird that this is possible with nbits > 0. + return fmt.Errorf("newState (%d) == oldState (%d) and no bits", newState, u) + } + s.decTable[u].newState = newState + } + } + return nil +} + +// decompress will decompress the bitstream. +// If the buffer is over-read an error is returned. +func (s *Scratch) decompress() error { + br := &s.bits + br.init(s.br.unread()) + + var s1, s2 decoder + // Initialize and decode first state and symbol. + s1.init(br, s.decTable, s.actualTableLog) + s2.init(br, s.decTable, s.actualTableLog) + + // Use temp table to avoid bound checks/append penalty. + var tmp = s.ct.tableSymbol[:256] + var off uint8 + + // Main part + if !s.zeroBits { + for br.off >= 8 { + br.fillFast() + tmp[off+0] = s1.nextFast() + tmp[off+1] = s2.nextFast() + br.fillFast() + tmp[off+2] = s1.nextFast() + tmp[off+3] = s2.nextFast() + off += 4 + // When off is 0, we have overflowed and should write. + if off == 0 { + s.Out = append(s.Out, tmp...) + if len(s.Out) >= s.DecompressLimit { + return fmt.Errorf("output size (%d) > DecompressLimit (%d)", len(s.Out), s.DecompressLimit) + } + } + } + } else { + for br.off >= 8 { + br.fillFast() + tmp[off+0] = s1.next() + tmp[off+1] = s2.next() + br.fillFast() + tmp[off+2] = s1.next() + tmp[off+3] = s2.next() + off += 4 + if off == 0 { + s.Out = append(s.Out, tmp...) + // When off is 0, we have overflowed and should write. + if len(s.Out) >= s.DecompressLimit { + return fmt.Errorf("output size (%d) > DecompressLimit (%d)", len(s.Out), s.DecompressLimit) + } + } + } + } + s.Out = append(s.Out, tmp[:off]...) + + // Final bits, a bit more expensive check + for { + if s1.finished() { + s.Out = append(s.Out, s1.final(), s2.final()) + break + } + br.fill() + s.Out = append(s.Out, s1.next()) + if s2.finished() { + s.Out = append(s.Out, s2.final(), s1.final()) + break + } + s.Out = append(s.Out, s2.next()) + if len(s.Out) >= s.DecompressLimit { + return fmt.Errorf("output size (%d) > DecompressLimit (%d)", len(s.Out), s.DecompressLimit) + } + } + return br.close() +} + +// decoder keeps track of the current state and updates it from the bitstream. +type decoder struct { + state uint16 + br *bitReader + dt []decSymbol +} + +// init will initialize the decoder and read the first state from the stream. +func (d *decoder) init(in *bitReader, dt []decSymbol, tableLog uint8) { + d.dt = dt + d.br = in + d.state = uint16(in.getBits(tableLog)) +} + +// next returns the next symbol and sets the next state. +// At least tablelog bits must be available in the bit reader. +func (d *decoder) next() uint8 { + n := &d.dt[d.state] + lowBits := d.br.getBits(n.nbBits) + d.state = n.newState + lowBits + return n.symbol +} + +// finished returns true if all bits have been read from the bitstream +// and the next state would require reading bits from the input. +func (d *decoder) finished() bool { + return d.br.finished() && d.dt[d.state].nbBits > 0 +} + +// final returns the current state symbol without decoding the next. +func (d *decoder) final() uint8 { + return d.dt[d.state].symbol +} + +// nextFast returns the next symbol and sets the next state. +// This can only be used if no symbols are 0 bits. +// At least tablelog bits must be available in the bit reader. +func (d *decoder) nextFast() uint8 { + n := d.dt[d.state] + lowBits := d.br.getBitsFast(n.nbBits) + d.state = n.newState + lowBits + return n.symbol +} diff --git a/vendor/github.com/klauspost/compress/fse/fse.go b/vendor/github.com/klauspost/compress/fse/fse.go new file mode 100644 index 000000000..075357b5b --- /dev/null +++ b/vendor/github.com/klauspost/compress/fse/fse.go @@ -0,0 +1,143 @@ +// Copyright 2018 Klaus Post. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// Based on work Copyright (c) 2013, Yann Collet, released under BSD License. + +// Package fse provides Finite State Entropy encoding and decoding. +// +// Finite State Entropy encoding provides a fast near-optimal symbol encoding/decoding +// for byte blocks as implemented in zstd. +// +// See https://github.com/klauspost/compress/tree/master/fse for more information. +package fse + +import ( + "errors" + "fmt" + "math/bits" +) + +const ( + /*!MEMORY_USAGE : + * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) + * Increasing memory usage improves compression ratio + * Reduced memory usage can improve speed, due to cache effect + * Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ + maxMemoryUsage = 14 + defaultMemoryUsage = 13 + + maxTableLog = maxMemoryUsage - 2 + maxTablesize = 1 << maxTableLog + defaultTablelog = defaultMemoryUsage - 2 + minTablelog = 5 + maxSymbolValue = 255 +) + +var ( + // ErrIncompressible is returned when input is judged to be too hard to compress. + ErrIncompressible = errors.New("input is not compressible") + + // ErrUseRLE is returned from the compressor when the input is a single byte value repeated. + ErrUseRLE = errors.New("input is single value repeated") +) + +// Scratch provides temporary storage for compression and decompression. +type Scratch struct { + // Private + count [maxSymbolValue + 1]uint32 + norm [maxSymbolValue + 1]int16 + symbolLen uint16 // Length of active part of the symbol table. + actualTableLog uint8 // Selected tablelog. + br byteReader + bits bitReader + bw bitWriter + ct cTable // Compression tables. + decTable []decSymbol // Decompression table. + zeroBits bool // no bits has prob > 50%. + clearCount bool // clear count + maxCount int // count of the most probable symbol + + // Per block parameters. + // These can be used to override compression parameters of the block. + // Do not touch, unless you know what you are doing. + + // Out is output buffer. + // If the scratch is re-used before the caller is done processing the output, + // set this field to nil. + // Otherwise the output buffer will be re-used for next Compression/Decompression step + // and allocation will be avoided. + Out []byte + + // MaxSymbolValue will override the maximum symbol value of the next block. + MaxSymbolValue uint8 + + // TableLog will attempt to override the tablelog for the next block. + TableLog uint8 + + // DecompressLimit limits the maximum decoded size acceptable. + // If > 0 decompression will stop when approximately this many bytes + // has been decoded. + // If 0, maximum size will be 2GB. + DecompressLimit int +} + +// Histogram allows to populate the histogram and skip that step in the compression, +// It otherwise allows to inspect the histogram when compression is done. +// To indicate that you have populated the histogram call HistogramFinished +// with the value of the highest populated symbol, as well as the number of entries +// in the most populated entry. These are accepted at face value. +// The returned slice will always be length 256. +func (s *Scratch) Histogram() []uint32 { + return s.count[:] +} + +// HistogramFinished can be called to indicate that the histogram has been populated. +// maxSymbol is the index of the highest set symbol of the next data segment. +// maxCount is the number of entries in the most populated entry. +// These are accepted at face value. +func (s *Scratch) HistogramFinished(maxSymbol uint8, maxCount int) { + s.maxCount = maxCount + s.symbolLen = uint16(maxSymbol) + 1 + s.clearCount = maxCount != 0 +} + +// prepare will prepare and allocate scratch tables used for both compression and decompression. +func (s *Scratch) prepare(in []byte) (*Scratch, error) { + if s == nil { + s = &Scratch{} + } + if s.MaxSymbolValue == 0 { + s.MaxSymbolValue = 255 + } + if s.TableLog == 0 { + s.TableLog = defaultTablelog + } + if s.TableLog > maxTableLog { + return nil, fmt.Errorf("tableLog (%d) > maxTableLog (%d)", s.TableLog, maxTableLog) + } + if cap(s.Out) == 0 { + s.Out = make([]byte, 0, len(in)) + } + if s.clearCount && s.maxCount == 0 { + for i := range s.count { + s.count[i] = 0 + } + s.clearCount = false + } + s.br.init(in) + if s.DecompressLimit == 0 { + // Max size 2GB. + s.DecompressLimit = (2 << 30) - 1 + } + + return s, nil +} + +// tableStep returns the next table index. +func tableStep(tableSize uint32) uint32 { + return (tableSize >> 1) + (tableSize >> 3) + 3 +} + +func highBits(val uint32) (n uint32) { + return uint32(bits.Len32(val) - 1) +} diff --git a/vendor/github.com/klauspost/compress/huff0/.gitignore b/vendor/github.com/klauspost/compress/huff0/.gitignore new file mode 100644 index 000000000..b3d262958 --- /dev/null +++ b/vendor/github.com/klauspost/compress/huff0/.gitignore @@ -0,0 +1 @@ +/huff0-fuzz.zip diff --git a/vendor/github.com/klauspost/compress/huff0/README.md b/vendor/github.com/klauspost/compress/huff0/README.md new file mode 100644 index 000000000..f2f4ff843 --- /dev/null +++ b/vendor/github.com/klauspost/compress/huff0/README.md @@ -0,0 +1,87 @@ +# Huff0 entropy compression + +This package provides Huff0 encoding and decoding as used in zstd. + +[Huff0](https://github.com/Cyan4973/FiniteStateEntropy#new-generation-entropy-coders), +a Huffman codec designed for modern CPU, featuring OoO (Out of Order) operations on multiple ALU +(Arithmetic Logic Unit), achieving extremely fast compression and decompression speeds. + +This can be used for compressing input with a lot of similar input values to the smallest number of bytes. +This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders, +but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding. + +* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/huff0) + +THIS PACKAGE IS NOT CONSIDERED STABLE AND API OR ENCODING MAY CHANGE IN THE FUTURE. + +## News + + * Mar 2018: First implementation released. Consider this beta software for now. + +# Usage + +This package provides a low level interface that allows to compress single independent blocks. + +Each block is separate, and there is no built in integrity checks. +This means that the caller should keep track of block sizes and also do checksums if needed. + +Compressing a block is done via the [`Compress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress1X) and +[`Compress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress4X) functions. +You must provide input and will receive the output and maybe an error. + +These error values can be returned: + +| Error | Description | +|---------------------|-----------------------------------------------------------------------------| +| `` | Everything ok, output is returned | +| `ErrIncompressible` | Returned when input is judged to be too hard to compress | +| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated | +| `ErrTooBig` | Returned if the input block exceeds the maximum allowed size (128 Kib) | +| `(error)` | An internal error occurred. | + + +As can be seen above some of there are errors that will be returned even under normal operation so it is important to handle these. + +To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object +that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same +object can be used for both. + +Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this +you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output. + +The `Scratch` object will retain state that allows to re-use previous tables for encoding and decoding. + +## Tables and re-use + +Huff0 allows for reusing tables from the previous block to save space if that is expected to give better/faster results. + +The Scratch object allows you to set a [`ReusePolicy`](https://godoc.org/github.com/klauspost/compress/huff0#ReusePolicy) +that controls this behaviour. See the documentation for details. This can be altered between each block. + +Do however note that this information is *not* stored in the output block and it is up to the users of the package to +record whether [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable) should be called, +based on the boolean reported back from the CompressXX call. + +If you want to store the table separate from the data, you can access them as `OutData` and `OutTable` on the +[`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object. + +## Decompressing + +The first part of decoding is to initialize the decoding table through [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable). +This will initialize the decoding tables. +You can supply the complete block to `ReadTable` and it will return the data part of the block +which can be given to the decompressor. + +Decompressing is done by calling the [`Decompress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress1X) +or [`Decompress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress4X) function. + +You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back +your input was likely corrupted. + +It is important to note that a successful decoding does *not* mean your output matches your original input. +There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid. + +# Contributing + +Contributions are always welcome. Be aware that adding public functions will require good justification and breaking +changes will likely not be accepted. If in doubt open an issue before writing the PR. \ No newline at end of file diff --git a/vendor/github.com/klauspost/compress/huff0/bitreader.go b/vendor/github.com/klauspost/compress/huff0/bitreader.go new file mode 100644 index 000000000..7d0903c70 --- /dev/null +++ b/vendor/github.com/klauspost/compress/huff0/bitreader.go @@ -0,0 +1,115 @@ +// Copyright 2018 Klaus Post. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// Based on work Copyright (c) 2013, Yann Collet, released under BSD License. + +package huff0 + +import ( + "errors" + "io" +) + +// bitReader reads a bitstream in reverse. +// The last set bit indicates the start of the stream and is used +// for aligning the input. +type bitReader struct { + in []byte + off uint // next byte to read is at in[off - 1] + value uint64 + bitsRead uint8 +} + +// init initializes and resets the bit reader. +func (b *bitReader) init(in []byte) error { + if len(in) < 1 { + return errors.New("corrupt stream: too short") + } + b.in = in + b.off = uint(len(in)) + // The highest bit of the last byte indicates where to start + v := in[len(in)-1] + if v == 0 { + return errors.New("corrupt stream, did not find end of stream") + } + b.bitsRead = 64 + b.value = 0 + b.fill() + b.fill() + b.bitsRead += 8 - uint8(highBit32(uint32(v))) + return nil +} + +// getBits will return n bits. n can be 0. +func (b *bitReader) getBits(n uint8) uint16 { + if n == 0 || b.bitsRead >= 64 { + return 0 + } + return b.getBitsFast(n) +} + +// getBitsFast requires that at least one bit is requested every time. +// There are no checks if the buffer is filled. +func (b *bitReader) getBitsFast(n uint8) uint16 { + const regMask = 64 - 1 + v := uint16((b.value << (b.bitsRead & regMask)) >> ((regMask + 1 - n) & regMask)) + b.bitsRead += n + return v +} + +// peekBitsFast requires that at least one bit is requested every time. +// There are no checks if the buffer is filled. +func (b *bitReader) peekBitsFast(n uint8) uint16 { + const regMask = 64 - 1 + v := uint16((b.value << (b.bitsRead & regMask)) >> ((regMask + 1 - n) & regMask)) + return v +} + +// fillFast() will make sure at least 32 bits are available. +// There must be at least 4 bytes available. +func (b *bitReader) fillFast() { + if b.bitsRead < 32 { + return + } + // Do single re-slice to avoid bounds checks. + v := b.in[b.off-4 : b.off] + low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) + b.value = (b.value << 32) | uint64(low) + b.bitsRead -= 32 + b.off -= 4 +} + +// fill() will make sure at least 32 bits are available. +func (b *bitReader) fill() { + if b.bitsRead < 32 { + return + } + if b.off > 4 { + v := b.in[b.off-4 : b.off] + low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) + b.value = (b.value << 32) | uint64(low) + b.bitsRead -= 32 + b.off -= 4 + return + } + for b.off > 0 { + b.value = (b.value << 8) | uint64(b.in[b.off-1]) + b.bitsRead -= 8 + b.off-- + } +} + +// finished returns true if all bits have been read from the bit stream. +func (b *bitReader) finished() bool { + return b.off == 0 && b.bitsRead >= 64 +} + +// close the bitstream and returns an error if out-of-buffer reads occurred. +func (b *bitReader) close() error { + // Release reference. + b.in = nil + if b.bitsRead > 64 { + return io.ErrUnexpectedEOF + } + return nil +} diff --git a/vendor/github.com/klauspost/compress/huff0/bitwriter.go b/vendor/github.com/klauspost/compress/huff0/bitwriter.go new file mode 100644 index 000000000..bda4021ef --- /dev/null +++ b/vendor/github.com/klauspost/compress/huff0/bitwriter.go @@ -0,0 +1,197 @@ +// Copyright 2018 Klaus Post. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// Based on work Copyright (c) 2013, Yann Collet, released under BSD License. + +package huff0 + +import "fmt" + +// bitWriter will write bits. +// First bit will be LSB of the first byte of output. +type bitWriter struct { + bitContainer uint64 + nBits uint8 + out []byte +} + +// bitMask16 is bitmasks. Has extra to avoid bounds check. +var bitMask16 = [32]uint16{ + 0, 1, 3, 7, 0xF, 0x1F, + 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, + 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0xFFFF, + 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, + 0xFFFF, 0xFFFF} /* up to 16 bits */ + +// addBits16NC will add up to 16 bits. +// It will not check if there is space for them, +// so the caller must ensure that it has flushed recently. +func (b *bitWriter) addBits16NC(value uint16, bits uint8) { + b.bitContainer |= uint64(value&bitMask16[bits&31]) << (b.nBits & 63) + b.nBits += bits +} + +// addBits16Clean will add up to 16 bits. value may not contain more set bits than indicated. +// It will not check if there is space for them, so the caller must ensure that it has flushed recently. +func (b *bitWriter) addBits16Clean(value uint16, bits uint8) { + b.bitContainer |= uint64(value) << (b.nBits & 63) + b.nBits += bits +} + +// encSymbol will add up to 16 bits. value may not contain more set bits than indicated. +// It will not check if there is space for them, so the caller must ensure that it has flushed recently. +func (b *bitWriter) encSymbol(ct cTable, symbol byte) { + enc := ct[symbol] + b.bitContainer |= uint64(enc.val) << (b.nBits & 63) + b.nBits += enc.nBits +} + +// encTwoSymbols will add up to 32 bits. value may not contain more set bits than indicated. +// It will not check if there is space for them, so the caller must ensure that it has flushed recently. +func (b *bitWriter) encTwoSymbols(ct cTable, av, bv byte) { + encA := ct[av] + encB := ct[bv] + sh := b.nBits & 63 + combined := uint64(encA.val) | (uint64(encB.val) << (encA.nBits & 63)) + b.bitContainer |= combined << sh + b.nBits += encA.nBits + encB.nBits +} + +// addBits16ZeroNC will add up to 16 bits. +// It will not check if there is space for them, +// so the caller must ensure that it has flushed recently. +// This is fastest if bits can be zero. +func (b *bitWriter) addBits16ZeroNC(value uint16, bits uint8) { + if bits == 0 { + return + } + value <<= (16 - bits) & 15 + value >>= (16 - bits) & 15 + b.bitContainer |= uint64(value) << (b.nBits & 63) + b.nBits += bits +} + +// flush will flush all pending full bytes. +// There will be at least 56 bits available for writing when this has been called. +// Using flush32 is faster, but leaves less space for writing. +func (b *bitWriter) flush() { + v := b.nBits >> 3 + switch v { + case 0: + return + case 1: + b.out = append(b.out, + byte(b.bitContainer), + ) + b.bitContainer >>= 1 << 3 + case 2: + b.out = append(b.out, + byte(b.bitContainer), + byte(b.bitContainer>>8), + ) + b.bitContainer >>= 2 << 3 + case 3: + b.out = append(b.out, + byte(b.bitContainer), + byte(b.bitContainer>>8), + byte(b.bitContainer>>16), + ) + b.bitContainer >>= 3 << 3 + case 4: + b.out = append(b.out, + byte(b.bitContainer), + byte(b.bitContainer>>8), + byte(b.bitContainer>>16), + byte(b.bitContainer>>24), + ) + b.bitContainer >>= 4 << 3 + case 5: + b.out = append(b.out, + byte(b.bitContainer), + byte(b.bitContainer>>8), + byte(b.bitContainer>>16), + byte(b.bitContainer>>24), + byte(b.bitContainer>>32), + ) + b.bitContainer >>= 5 << 3 + case 6: + b.out = append(b.out, + byte(b.bitContainer), + byte(b.bitContainer>>8), + byte(b.bitContainer>>16), + byte(b.bitContainer>>24), + byte(b.bitContainer>>32), + byte(b.bitContainer>>40), + ) + b.bitContainer >>= 6 << 3 + case 7: + b.out = append(b.out, + byte(b.bitContainer), + byte(b.bitContainer>>8), + byte(b.bitContainer>>16), + byte(b.bitContainer>>24), + byte(b.bitContainer>>32), + byte(b.bitContainer>>40), + byte(b.bitContainer>>48), + ) + b.bitContainer >>= 7 << 3 + case 8: + b.out = append(b.out, + byte(b.bitContainer), + byte(b.bitContainer>>8), + byte(b.bitContainer>>16), + byte(b.bitContainer>>24), + byte(b.bitContainer>>32), + byte(b.bitContainer>>40), + byte(b.bitContainer>>48), + byte(b.bitContainer>>56), + ) + b.bitContainer = 0 + b.nBits = 0 + return + default: + panic(fmt.Errorf("bits (%d) > 64", b.nBits)) + } + b.nBits &= 7 +} + +// flush32 will flush out, so there are at least 32 bits available for writing. +func (b *bitWriter) flush32() { + if b.nBits < 32 { + return + } + b.out = append(b.out, + byte(b.bitContainer), + byte(b.bitContainer>>8), + byte(b.bitContainer>>16), + byte(b.bitContainer>>24)) + b.nBits -= 32 + b.bitContainer >>= 32 +} + +// flushAlign will flush remaining full bytes and align to next byte boundary. +func (b *bitWriter) flushAlign() { + nbBytes := (b.nBits + 7) >> 3 + for i := uint8(0); i < nbBytes; i++ { + b.out = append(b.out, byte(b.bitContainer>>(i*8))) + } + b.nBits = 0 + b.bitContainer = 0 +} + +// close will write the alignment bit and write the final byte(s) +// to the output. +func (b *bitWriter) close() error { + // End mark + b.addBits16Clean(1, 1) + // flush until next byte. + b.flushAlign() + return nil +} + +// reset and continue writing by appending to out. +func (b *bitWriter) reset(out []byte) { + b.bitContainer = 0 + b.nBits = 0 + b.out = out +} diff --git a/vendor/github.com/klauspost/compress/huff0/bytereader.go b/vendor/github.com/klauspost/compress/huff0/bytereader.go new file mode 100644 index 000000000..50bcdf6ea --- /dev/null +++ b/vendor/github.com/klauspost/compress/huff0/bytereader.go @@ -0,0 +1,54 @@ +// Copyright 2018 Klaus Post. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// Based on work Copyright (c) 2013, Yann Collet, released under BSD License. + +package huff0 + +// byteReader provides a byte reader that reads +// little endian values from a byte stream. +// The input stream is manually advanced. +// The reader performs no bounds checks. +type byteReader struct { + b []byte + off int +} + +// init will initialize the reader and set the input. +func (b *byteReader) init(in []byte) { + b.b = in + b.off = 0 +} + +// advance the stream b n bytes. +func (b *byteReader) advance(n uint) { + b.off += int(n) +} + +// Int32 returns a little endian int32 starting at current offset. +func (b byteReader) Int32() int32 { + v3 := int32(b.b[b.off+3]) + v2 := int32(b.b[b.off+2]) + v1 := int32(b.b[b.off+1]) + v0 := int32(b.b[b.off]) + return (v3 << 24) | (v2 << 16) | (v1 << 8) | v0 +} + +// Uint32 returns a little endian uint32 starting at current offset. +func (b byteReader) Uint32() uint32 { + v3 := uint32(b.b[b.off+3]) + v2 := uint32(b.b[b.off+2]) + v1 := uint32(b.b[b.off+1]) + v0 := uint32(b.b[b.off]) + return (v3 << 24) | (v2 << 16) | (v1 << 8) | v0 +} + +// unread returns the unread portion of the input. +func (b byteReader) unread() []byte { + return b.b[b.off:] +} + +// remain will return the number of bytes remaining. +func (b byteReader) remain() int { + return len(b.b) - b.off +} diff --git a/vendor/github.com/klauspost/compress/huff0/compress.go b/vendor/github.com/klauspost/compress/huff0/compress.go new file mode 100644 index 000000000..125429d74 --- /dev/null +++ b/vendor/github.com/klauspost/compress/huff0/compress.go @@ -0,0 +1,625 @@ +package huff0 + +import ( + "fmt" + "runtime" + "sync" +) + +// Compress1X will compress the input. +// The output can be decoded using Decompress1X. +// Supply a Scratch object. The scratch object contains state about re-use, +// So when sharing across independent encodes, be sure to set the re-use policy. +func Compress1X(in []byte, s *Scratch) (out []byte, reUsed bool, err error) { + s, err = s.prepare(in) + if err != nil { + return nil, false, err + } + return compress(in, s, s.compress1X) +} + +// Compress4X will compress the input. The input is split into 4 independent blocks +// and compressed similar to Compress1X. +// The output can be decoded using Decompress4X. +// Supply a Scratch object. The scratch object contains state about re-use, +// So when sharing across independent encodes, be sure to set the re-use policy. +func Compress4X(in []byte, s *Scratch) (out []byte, reUsed bool, err error) { + s, err = s.prepare(in) + if err != nil { + return nil, false, err + } + if false { + // TODO: compress4Xp only slightly faster. + const parallelThreshold = 8 << 10 + if len(in) < parallelThreshold || runtime.GOMAXPROCS(0) == 1 { + return compress(in, s, s.compress4X) + } + return compress(in, s, s.compress4Xp) + } + return compress(in, s, s.compress4X) +} + +func compress(in []byte, s *Scratch, compressor func(src []byte) ([]byte, error)) (out []byte, reUsed bool, err error) { + // Nuke previous table if we cannot reuse anyway. + if s.Reuse == ReusePolicyNone { + s.prevTable = s.prevTable[:0] + } + + // Create histogram, if none was provided. + maxCount := s.maxCount + var canReuse = false + if maxCount == 0 { + maxCount, canReuse = s.countSimple(in) + } else { + canReuse = s.canUseTable(s.prevTable) + } + + // We want the output size to be less than this: + wantSize := len(in) + if s.WantLogLess > 0 { + wantSize -= wantSize >> s.WantLogLess + } + + // Reset for next run. + s.clearCount = true + s.maxCount = 0 + if maxCount >= len(in) { + if maxCount > len(in) { + return nil, false, fmt.Errorf("maxCount (%d) > length (%d)", maxCount, len(in)) + } + if len(in) == 1 { + return nil, false, ErrIncompressible + } + // One symbol, use RLE + return nil, false, ErrUseRLE + } + if maxCount == 1 || maxCount < (len(in)>>7) { + // Each symbol present maximum once or too well distributed. + return nil, false, ErrIncompressible + } + + if s.Reuse == ReusePolicyPrefer && canReuse { + keepTable := s.cTable + s.cTable = s.prevTable + s.Out, err = compressor(in) + s.cTable = keepTable + if err == nil && len(s.Out) < wantSize { + s.OutData = s.Out + return s.Out, true, nil + } + // Do not attempt to re-use later. + s.prevTable = s.prevTable[:0] + } + + // Calculate new table. + s.optimalTableLog() + err = s.buildCTable() + if err != nil { + return nil, false, err + } + + if false && !s.canUseTable(s.cTable) { + panic("invalid table generated") + } + + if s.Reuse == ReusePolicyAllow && canReuse { + hSize := len(s.Out) + oldSize := s.prevTable.estimateSize(s.count[:s.symbolLen]) + newSize := s.cTable.estimateSize(s.count[:s.symbolLen]) + if oldSize <= hSize+newSize || hSize+12 >= wantSize { + // Retain cTable even if we re-use. + keepTable := s.cTable + s.cTable = s.prevTable + s.Out, err = compressor(in) + s.cTable = keepTable + if err != nil { + return nil, false, err + } + if len(s.Out) >= wantSize { + return nil, false, ErrIncompressible + } + s.OutData = s.Out + return s.Out, true, nil + } + } + + // Use new table + err = s.cTable.write(s) + if err != nil { + s.OutTable = nil + return nil, false, err + } + s.OutTable = s.Out + + // Compress using new table + s.Out, err = compressor(in) + if err != nil { + s.OutTable = nil + return nil, false, err + } + if len(s.Out) >= wantSize { + s.OutTable = nil + return nil, false, ErrIncompressible + } + // Move current table into previous. + s.prevTable, s.cTable = s.cTable, s.prevTable[:0] + s.OutData = s.Out[len(s.OutTable):] + return s.Out, false, nil +} + +func (s *Scratch) compress1X(src []byte) ([]byte, error) { + return s.compress1xDo(s.Out, src) +} + +func (s *Scratch) compress1xDo(dst, src []byte) ([]byte, error) { + var bw = bitWriter{out: dst} + + // N is length divisible by 4. + n := len(src) + n -= n & 3 + cTable := s.cTable[:256] + + // Encode last bytes. + for i := len(src) & 3; i > 0; i-- { + bw.encSymbol(cTable, src[n+i-1]) + } + n -= 4 + if s.actualTableLog <= 8 { + for ; n >= 0; n -= 4 { + tmp := src[n : n+4] + // tmp should be len 4 + bw.flush32() + bw.encTwoSymbols(cTable, tmp[3], tmp[2]) + bw.encTwoSymbols(cTable, tmp[1], tmp[0]) + } + } else { + for ; n >= 0; n -= 4 { + tmp := src[n : n+4] + // tmp should be len 4 + bw.flush32() + bw.encTwoSymbols(cTable, tmp[3], tmp[2]) + bw.flush32() + bw.encTwoSymbols(cTable, tmp[1], tmp[0]) + } + } + err := bw.close() + return bw.out, err +} + +var sixZeros [6]byte + +func (s *Scratch) compress4X(src []byte) ([]byte, error) { + if len(src) < 12 { + return nil, ErrIncompressible + } + segmentSize := (len(src) + 3) / 4 + + // Add placeholder for output length + offsetIdx := len(s.Out) + s.Out = append(s.Out, sixZeros[:]...) + + for i := 0; i < 4; i++ { + toDo := src + if len(toDo) > segmentSize { + toDo = toDo[:segmentSize] + } + src = src[len(toDo):] + + var err error + idx := len(s.Out) + s.Out, err = s.compress1xDo(s.Out, toDo) + if err != nil { + return nil, err + } + // Write compressed length as little endian before block. + if i < 3 { + // Last length is not written. + length := len(s.Out) - idx + s.Out[i*2+offsetIdx] = byte(length) + s.Out[i*2+offsetIdx+1] = byte(length >> 8) + } + } + + return s.Out, nil +} + +// compress4Xp will compress 4 streams using separate goroutines. +func (s *Scratch) compress4Xp(src []byte) ([]byte, error) { + if len(src) < 12 { + return nil, ErrIncompressible + } + // Add placeholder for output length + s.Out = s.Out[:6] + + segmentSize := (len(src) + 3) / 4 + var wg sync.WaitGroup + var errs [4]error + wg.Add(4) + for i := 0; i < 4; i++ { + toDo := src + if len(toDo) > segmentSize { + toDo = toDo[:segmentSize] + } + src = src[len(toDo):] + + // Separate goroutine for each block. + go func(i int) { + s.tmpOut[i], errs[i] = s.compress1xDo(s.tmpOut[i][:0], toDo) + wg.Done() + }(i) + } + wg.Wait() + for i := 0; i < 4; i++ { + if errs[i] != nil { + return nil, errs[i] + } + o := s.tmpOut[i] + // Write compressed length as little endian before block. + if i < 3 { + // Last length is not written. + s.Out[i*2] = byte(len(o)) + s.Out[i*2+1] = byte(len(o) >> 8) + } + + // Write output. + s.Out = append(s.Out, o...) + } + return s.Out, nil +} + +// countSimple will create a simple histogram in s.count. +// Returns the biggest count. +// Does not update s.clearCount. +func (s *Scratch) countSimple(in []byte) (max int, reuse bool) { + reuse = true + for _, v := range in { + s.count[v]++ + } + m := uint32(0) + if len(s.prevTable) > 0 { + for i, v := range s.count[:] { + if v > m { + m = v + } + if v > 0 { + s.symbolLen = uint16(i) + 1 + if i >= len(s.prevTable) { + reuse = false + } else { + if s.prevTable[i].nBits == 0 { + reuse = false + } + } + } + } + return int(m), reuse + } + for i, v := range s.count[:] { + if v > m { + m = v + } + if v > 0 { + s.symbolLen = uint16(i) + 1 + } + } + return int(m), false +} + +func (s *Scratch) canUseTable(c cTable) bool { + if len(c) < int(s.symbolLen) { + return false + } + for i, v := range s.count[:s.symbolLen] { + if v != 0 && c[i].nBits == 0 { + return false + } + } + return true +} + +// minTableLog provides the minimum logSize to safely represent a distribution. +func (s *Scratch) minTableLog() uint8 { + minBitsSrc := highBit32(uint32(s.br.remain()-1)) + 1 + minBitsSymbols := highBit32(uint32(s.symbolLen-1)) + 2 + if minBitsSrc < minBitsSymbols { + return uint8(minBitsSrc) + } + return uint8(minBitsSymbols) +} + +// optimalTableLog calculates and sets the optimal tableLog in s.actualTableLog +func (s *Scratch) optimalTableLog() { + tableLog := s.TableLog + minBits := s.minTableLog() + maxBitsSrc := uint8(highBit32(uint32(s.br.remain()-1))) - 2 + if maxBitsSrc < tableLog { + // Accuracy can be reduced + tableLog = maxBitsSrc + } + if minBits > tableLog { + tableLog = minBits + } + // Need a minimum to safely represent all symbol values + if tableLog < minTablelog { + tableLog = minTablelog + } + if tableLog > tableLogMax { + tableLog = tableLogMax + } + s.actualTableLog = tableLog +} + +type cTableEntry struct { + val uint16 + nBits uint8 + // We have 8 bits extra +} + +const huffNodesMask = huffNodesLen - 1 + +func (s *Scratch) buildCTable() error { + s.huffSort() + if cap(s.cTable) < maxSymbolValue+1 { + s.cTable = make([]cTableEntry, s.symbolLen, maxSymbolValue+1) + } else { + s.cTable = s.cTable[:s.symbolLen] + for i := range s.cTable { + s.cTable[i] = cTableEntry{} + } + } + + var startNode = int16(s.symbolLen) + nonNullRank := s.symbolLen - 1 + + nodeNb := int16(startNode) + huffNode := s.nodes[1 : huffNodesLen+1] + + // This overlays the slice above, but allows "-1" index lookups. + // Different from reference implementation. + huffNode0 := s.nodes[0 : huffNodesLen+1] + + for huffNode[nonNullRank].count == 0 { + nonNullRank-- + } + + lowS := int16(nonNullRank) + nodeRoot := nodeNb + lowS - 1 + lowN := nodeNb + huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count + huffNode[lowS].parent, huffNode[lowS-1].parent = uint16(nodeNb), uint16(nodeNb) + nodeNb++ + lowS -= 2 + for n := nodeNb; n <= nodeRoot; n++ { + huffNode[n].count = 1 << 30 + } + // fake entry, strong barrier + huffNode0[0].count = 1 << 31 + + // create parents + for nodeNb <= nodeRoot { + var n1, n2 int16 + if huffNode0[lowS+1].count < huffNode0[lowN+1].count { + n1 = lowS + lowS-- + } else { + n1 = lowN + lowN++ + } + if huffNode0[lowS+1].count < huffNode0[lowN+1].count { + n2 = lowS + lowS-- + } else { + n2 = lowN + lowN++ + } + + huffNode[nodeNb].count = huffNode0[n1+1].count + huffNode0[n2+1].count + huffNode0[n1+1].parent, huffNode0[n2+1].parent = uint16(nodeNb), uint16(nodeNb) + nodeNb++ + } + + // distribute weights (unlimited tree height) + huffNode[nodeRoot].nbBits = 0 + for n := nodeRoot - 1; n >= startNode; n-- { + huffNode[n].nbBits = huffNode[huffNode[n].parent].nbBits + 1 + } + for n := uint16(0); n <= nonNullRank; n++ { + huffNode[n].nbBits = huffNode[huffNode[n].parent].nbBits + 1 + } + s.actualTableLog = s.setMaxHeight(int(nonNullRank)) + maxNbBits := s.actualTableLog + + // fill result into tree (val, nbBits) + if maxNbBits > tableLogMax { + return fmt.Errorf("internal error: maxNbBits (%d) > tableLogMax (%d)", maxNbBits, tableLogMax) + } + var nbPerRank [tableLogMax + 1]uint16 + var valPerRank [16]uint16 + for _, v := range huffNode[:nonNullRank+1] { + nbPerRank[v.nbBits]++ + } + // determine stating value per rank + { + min := uint16(0) + for n := maxNbBits; n > 0; n-- { + // get starting value within each rank + valPerRank[n] = min + min += nbPerRank[n] + min >>= 1 + } + } + + // push nbBits per symbol, symbol order + for _, v := range huffNode[:nonNullRank+1] { + s.cTable[v.symbol].nBits = v.nbBits + } + + // assign value within rank, symbol order + t := s.cTable[:s.symbolLen] + for n, val := range t { + nbits := val.nBits & 15 + v := valPerRank[nbits] + t[n].val = v + valPerRank[nbits] = v + 1 + } + + return nil +} + +// huffSort will sort symbols, decreasing order. +func (s *Scratch) huffSort() { + type rankPos struct { + base uint32 + current uint32 + } + + // Clear nodes + nodes := s.nodes[:huffNodesLen+1] + s.nodes = nodes + nodes = nodes[1 : huffNodesLen+1] + + // Sort into buckets based on length of symbol count. + var rank [32]rankPos + for _, v := range s.count[:s.symbolLen] { + r := highBit32(v+1) & 31 + rank[r].base++ + } + // maxBitLength is log2(BlockSizeMax) + 1 + const maxBitLength = 18 + 1 + for n := maxBitLength; n > 0; n-- { + rank[n-1].base += rank[n].base + } + for n := range rank[:maxBitLength] { + rank[n].current = rank[n].base + } + for n, c := range s.count[:s.symbolLen] { + r := (highBit32(c+1) + 1) & 31 + pos := rank[r].current + rank[r].current++ + prev := nodes[(pos-1)&huffNodesMask] + for pos > rank[r].base && c > prev.count { + nodes[pos&huffNodesMask] = prev + pos-- + prev = nodes[(pos-1)&huffNodesMask] + } + nodes[pos&huffNodesMask] = nodeElt{count: c, symbol: byte(n)} + } + return +} + +func (s *Scratch) setMaxHeight(lastNonNull int) uint8 { + maxNbBits := s.actualTableLog + huffNode := s.nodes[1 : huffNodesLen+1] + //huffNode = huffNode[: huffNodesLen] + + largestBits := huffNode[lastNonNull].nbBits + + // early exit : no elt > maxNbBits + if largestBits <= maxNbBits { + return largestBits + } + totalCost := int(0) + baseCost := int(1) << (largestBits - maxNbBits) + n := uint32(lastNonNull) + + for huffNode[n].nbBits > maxNbBits { + totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)) + huffNode[n].nbBits = maxNbBits + n-- + } + // n stops at huffNode[n].nbBits <= maxNbBits + + for huffNode[n].nbBits == maxNbBits { + n-- + } + // n end at index of smallest symbol using < maxNbBits + + // renorm totalCost + totalCost >>= largestBits - maxNbBits /* note : totalCost is necessarily a multiple of baseCost */ + + // repay normalized cost + { + const noSymbol = 0xF0F0F0F0 + var rankLast [tableLogMax + 2]uint32 + + for i := range rankLast[:] { + rankLast[i] = noSymbol + } + + // Get pos of last (smallest) symbol per rank + { + currentNbBits := uint8(maxNbBits) + for pos := int(n); pos >= 0; pos-- { + if huffNode[pos].nbBits >= currentNbBits { + continue + } + currentNbBits = huffNode[pos].nbBits // < maxNbBits + rankLast[maxNbBits-currentNbBits] = uint32(pos) + } + } + + for totalCost > 0 { + nBitsToDecrease := uint8(highBit32(uint32(totalCost))) + 1 + + for ; nBitsToDecrease > 1; nBitsToDecrease-- { + highPos := rankLast[nBitsToDecrease] + lowPos := rankLast[nBitsToDecrease-1] + if highPos == noSymbol { + continue + } + if lowPos == noSymbol { + break + } + highTotal := huffNode[highPos].count + lowTotal := 2 * huffNode[lowPos].count + if highTotal <= lowTotal { + break + } + } + // only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) + // HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary + // FIXME: try to remove + for (nBitsToDecrease <= tableLogMax) && (rankLast[nBitsToDecrease] == noSymbol) { + nBitsToDecrease++ + } + totalCost -= 1 << (nBitsToDecrease - 1) + if rankLast[nBitsToDecrease-1] == noSymbol { + // this rank is no longer empty + rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease] + } + huffNode[rankLast[nBitsToDecrease]].nbBits++ + if rankLast[nBitsToDecrease] == 0 { + /* special case, reached largest symbol */ + rankLast[nBitsToDecrease] = noSymbol + } else { + rankLast[nBitsToDecrease]-- + if huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease { + rankLast[nBitsToDecrease] = noSymbol /* this rank is now empty */ + } + } + } + + for totalCost < 0 { /* Sometimes, cost correction overshoot */ + if rankLast[1] == noSymbol { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */ + for huffNode[n].nbBits == maxNbBits { + n-- + } + huffNode[n+1].nbBits-- + rankLast[1] = n + 1 + totalCost++ + continue + } + huffNode[rankLast[1]+1].nbBits-- + rankLast[1]++ + totalCost++ + } + } + return maxNbBits +} + +type nodeElt struct { + count uint32 + parent uint16 + symbol byte + nbBits uint8 +} diff --git a/vendor/github.com/klauspost/compress/huff0/decompress.go b/vendor/github.com/klauspost/compress/huff0/decompress.go new file mode 100644 index 000000000..97ae66a4a --- /dev/null +++ b/vendor/github.com/klauspost/compress/huff0/decompress.go @@ -0,0 +1,472 @@ +package huff0 + +import ( + "errors" + "fmt" + "io" + + "github.com/klauspost/compress/fse" +) + +type dTable struct { + single []dEntrySingle + double []dEntryDouble +} + +// single-symbols decoding +type dEntrySingle struct { + entry uint16 +} + +// double-symbols decoding +type dEntryDouble struct { + seq uint16 + nBits uint8 + len uint8 +} + +// ReadTable will read a table from the input. +// The size of the input may be larger than the table definition. +// Any content remaining after the table definition will be returned. +// If no Scratch is provided a new one is allocated. +// The returned Scratch can be used for decoding input using this table. +func ReadTable(in []byte, s *Scratch) (s2 *Scratch, remain []byte, err error) { + s, err = s.prepare(in) + if err != nil { + return s, nil, err + } + if len(in) <= 1 { + return s, nil, errors.New("input too small for table") + } + iSize := in[0] + in = in[1:] + if iSize >= 128 { + // Uncompressed + oSize := iSize - 127 + iSize = (oSize + 1) / 2 + if int(iSize) > len(in) { + return s, nil, errors.New("input too small for table") + } + for n := uint8(0); n < oSize; n += 2 { + v := in[n/2] + s.huffWeight[n] = v >> 4 + s.huffWeight[n+1] = v & 15 + } + s.symbolLen = uint16(oSize) + in = in[iSize:] + } else { + if len(in) <= int(iSize) { + return s, nil, errors.New("input too small for table") + } + // FSE compressed weights + s.fse.DecompressLimit = 255 + hw := s.huffWeight[:] + s.fse.Out = hw + b, err := fse.Decompress(in[:iSize], s.fse) + s.fse.Out = nil + if err != nil { + return s, nil, err + } + if len(b) > 255 { + return s, nil, errors.New("corrupt input: output table too large") + } + s.symbolLen = uint16(len(b)) + in = in[iSize:] + } + + // collect weight stats + var rankStats [16]uint32 + weightTotal := uint32(0) + for _, v := range s.huffWeight[:s.symbolLen] { + if v > tableLogMax { + return s, nil, errors.New("corrupt input: weight too large") + } + v2 := v & 15 + rankStats[v2]++ + weightTotal += (1 << v2) >> 1 + } + if weightTotal == 0 { + return s, nil, errors.New("corrupt input: weights zero") + } + + // get last non-null symbol weight (implied, total must be 2^n) + { + tableLog := highBit32(weightTotal) + 1 + if tableLog > tableLogMax { + return s, nil, errors.New("corrupt input: tableLog too big") + } + s.actualTableLog = uint8(tableLog) + // determine last weight + { + total := uint32(1) << tableLog + rest := total - weightTotal + verif := uint32(1) << highBit32(rest) + lastWeight := highBit32(rest) + 1 + if verif != rest { + // last value must be a clean power of 2 + return s, nil, errors.New("corrupt input: last value not power of two") + } + s.huffWeight[s.symbolLen] = uint8(lastWeight) + s.symbolLen++ + rankStats[lastWeight]++ + } + } + + if (rankStats[1] < 2) || (rankStats[1]&1 != 0) { + // by construction : at least 2 elts of rank 1, must be even + return s, nil, errors.New("corrupt input: min elt size, even check failed ") + } + + // TODO: Choose between single/double symbol decoding + + // Calculate starting value for each rank + { + var nextRankStart uint32 + for n := uint8(1); n < s.actualTableLog+1; n++ { + current := nextRankStart + nextRankStart += rankStats[n] << (n - 1) + rankStats[n] = current + } + } + + // fill DTable (always full size) + tSize := 1 << tableLogMax + if len(s.dt.single) != tSize { + s.dt.single = make([]dEntrySingle, tSize) + } + for n, w := range s.huffWeight[:s.symbolLen] { + if w == 0 { + continue + } + length := (uint32(1) << w) >> 1 + d := dEntrySingle{ + entry: uint16(s.actualTableLog+1-w) | (uint16(n) << 8), + } + single := s.dt.single[rankStats[w] : rankStats[w]+length] + for i := range single { + single[i] = d + } + rankStats[w] += length + } + return s, in, nil +} + +// Decompress1X will decompress a 1X encoded stream. +// The length of the supplied input must match the end of a block exactly. +// Before this is called, the table must be initialized with ReadTable unless +// the encoder re-used the table. +func (s *Scratch) Decompress1X(in []byte) (out []byte, err error) { + if len(s.dt.single) == 0 { + return nil, errors.New("no table loaded") + } + var br bitReader + err = br.init(in) + if err != nil { + return nil, err + } + s.Out = s.Out[:0] + + decode := func() byte { + val := br.peekBitsFast(s.actualTableLog) /* note : actualTableLog >= 1 */ + v := s.dt.single[val] + br.bitsRead += uint8(v.entry) + return uint8(v.entry >> 8) + } + hasDec := func(v dEntrySingle) byte { + br.bitsRead += uint8(v.entry) + return uint8(v.entry >> 8) + } + + // Avoid bounds check by always having full sized table. + const tlSize = 1 << tableLogMax + const tlMask = tlSize - 1 + dt := s.dt.single[:tlSize] + + // Use temp table to avoid bound checks/append penalty. + var tmp = s.huffWeight[:256] + var off uint8 + + for br.off >= 8 { + br.fillFast() + tmp[off+0] = hasDec(dt[br.peekBitsFast(s.actualTableLog)&tlMask]) + tmp[off+1] = hasDec(dt[br.peekBitsFast(s.actualTableLog)&tlMask]) + br.fillFast() + tmp[off+2] = hasDec(dt[br.peekBitsFast(s.actualTableLog)&tlMask]) + tmp[off+3] = hasDec(dt[br.peekBitsFast(s.actualTableLog)&tlMask]) + off += 4 + if off == 0 { + if len(s.Out)+256 > s.MaxDecodedSize { + br.close() + return nil, ErrMaxDecodedSizeExceeded + } + s.Out = append(s.Out, tmp...) + } + } + + if len(s.Out)+int(off) > s.MaxDecodedSize { + br.close() + return nil, ErrMaxDecodedSizeExceeded + } + s.Out = append(s.Out, tmp[:off]...) + + for !br.finished() { + br.fill() + if len(s.Out) >= s.MaxDecodedSize { + br.close() + return nil, ErrMaxDecodedSizeExceeded + } + s.Out = append(s.Out, decode()) + } + return s.Out, br.close() +} + +// Decompress4X will decompress a 4X encoded stream. +// Before this is called, the table must be initialized with ReadTable unless +// the encoder re-used the table. +// The length of the supplied input must match the end of a block exactly. +// The destination size of the uncompressed data must be known and provided. +func (s *Scratch) Decompress4X(in []byte, dstSize int) (out []byte, err error) { + if len(s.dt.single) == 0 { + return nil, errors.New("no table loaded") + } + if len(in) < 6+(4*1) { + return nil, errors.New("input too small") + } + if dstSize > s.MaxDecodedSize { + return nil, ErrMaxDecodedSizeExceeded + } + // TODO: We do not detect when we overrun a buffer, except if the last one does. + + var br [4]bitReader + start := 6 + for i := 0; i < 3; i++ { + length := int(in[i*2]) | (int(in[i*2+1]) << 8) + if start+length >= len(in) { + return nil, errors.New("truncated input (or invalid offset)") + } + err = br[i].init(in[start : start+length]) + if err != nil { + return nil, err + } + start += length + } + err = br[3].init(in[start:]) + if err != nil { + return nil, err + } + + // Prepare output + if cap(s.Out) < dstSize { + s.Out = make([]byte, 0, dstSize) + } + s.Out = s.Out[:dstSize] + // destination, offset to match first output + dstOut := s.Out + dstEvery := (dstSize + 3) / 4 + + const tlSize = 1 << tableLogMax + const tlMask = tlSize - 1 + single := s.dt.single[:tlSize] + + decode := func(br *bitReader) byte { + val := br.peekBitsFast(s.actualTableLog) /* note : actualTableLog >= 1 */ + v := single[val&tlMask] + br.bitsRead += uint8(v.entry) + return uint8(v.entry >> 8) + } + + // Use temp table to avoid bound checks/append penalty. + var tmp = s.huffWeight[:256] + var off uint8 + var decoded int + + // Decode 2 values from each decoder/loop. + const bufoff = 256 / 4 +bigloop: + for { + for i := range br { + br := &br[i] + if br.off < 4 { + break bigloop + } + br.fillFast() + } + + { + const stream = 0 + val := br[stream].peekBitsFast(s.actualTableLog) + v := single[val&tlMask] + br[stream].bitsRead += uint8(v.entry) + + val2 := br[stream].peekBitsFast(s.actualTableLog) + v2 := single[val2&tlMask] + tmp[off+bufoff*stream+1] = uint8(v2.entry >> 8) + tmp[off+bufoff*stream] = uint8(v.entry >> 8) + br[stream].bitsRead += uint8(v2.entry) + } + + { + const stream = 1 + val := br[stream].peekBitsFast(s.actualTableLog) + v := single[val&tlMask] + br[stream].bitsRead += uint8(v.entry) + + val2 := br[stream].peekBitsFast(s.actualTableLog) + v2 := single[val2&tlMask] + tmp[off+bufoff*stream+1] = uint8(v2.entry >> 8) + tmp[off+bufoff*stream] = uint8(v.entry >> 8) + br[stream].bitsRead += uint8(v2.entry) + } + + { + const stream = 2 + val := br[stream].peekBitsFast(s.actualTableLog) + v := single[val&tlMask] + br[stream].bitsRead += uint8(v.entry) + + val2 := br[stream].peekBitsFast(s.actualTableLog) + v2 := single[val2&tlMask] + tmp[off+bufoff*stream+1] = uint8(v2.entry >> 8) + tmp[off+bufoff*stream] = uint8(v.entry >> 8) + br[stream].bitsRead += uint8(v2.entry) + } + + { + const stream = 3 + val := br[stream].peekBitsFast(s.actualTableLog) + v := single[val&tlMask] + br[stream].bitsRead += uint8(v.entry) + + val2 := br[stream].peekBitsFast(s.actualTableLog) + v2 := single[val2&tlMask] + tmp[off+bufoff*stream+1] = uint8(v2.entry >> 8) + tmp[off+bufoff*stream] = uint8(v.entry >> 8) + br[stream].bitsRead += uint8(v2.entry) + } + + off += 2 + + if off == bufoff { + if bufoff > dstEvery { + return nil, errors.New("corruption detected: stream overrun 1") + } + copy(dstOut, tmp[:bufoff]) + copy(dstOut[dstEvery:], tmp[bufoff:bufoff*2]) + copy(dstOut[dstEvery*2:], tmp[bufoff*2:bufoff*3]) + copy(dstOut[dstEvery*3:], tmp[bufoff*3:bufoff*4]) + off = 0 + dstOut = dstOut[bufoff:] + decoded += 256 + // There must at least be 3 buffers left. + if len(dstOut) < dstEvery*3 { + return nil, errors.New("corruption detected: stream overrun 2") + } + } + } + if off > 0 { + ioff := int(off) + if len(dstOut) < dstEvery*3+ioff { + return nil, errors.New("corruption detected: stream overrun 3") + } + copy(dstOut, tmp[:off]) + copy(dstOut[dstEvery:dstEvery+ioff], tmp[bufoff:bufoff*2]) + copy(dstOut[dstEvery*2:dstEvery*2+ioff], tmp[bufoff*2:bufoff*3]) + copy(dstOut[dstEvery*3:dstEvery*3+ioff], tmp[bufoff*3:bufoff*4]) + decoded += int(off) * 4 + dstOut = dstOut[off:] + } + + // Decode remaining. + for i := range br { + offset := dstEvery * i + br := &br[i] + for !br.finished() { + br.fill() + if offset >= len(dstOut) { + return nil, errors.New("corruption detected: stream overrun 4") + } + dstOut[offset] = decode(br) + offset++ + } + decoded += offset - dstEvery*i + err = br.close() + if err != nil { + return nil, err + } + } + if dstSize != decoded { + return nil, errors.New("corruption detected: short output block") + } + return s.Out, nil +} + +// matches will compare a decoding table to a coding table. +// Errors are written to the writer. +// Nothing will be written if table is ok. +func (s *Scratch) matches(ct cTable, w io.Writer) { + if s == nil || len(s.dt.single) == 0 { + return + } + dt := s.dt.single[:1<>8) == byte(sym) { + fmt.Fprintf(w, "symbol %x has decoder, but no encoder\n", sym) + errs++ + break + } + } + if errs == 0 { + broken-- + } + continue + } + // Unused bits in input + ub := tablelog - enc.nBits + top := enc.val << ub + // decoder looks at top bits. + dec := dt[top] + if uint8(dec.entry) != enc.nBits { + fmt.Fprintf(w, "symbol 0x%x bit size mismatch (enc: %d, dec:%d).\n", sym, enc.nBits, uint8(dec.entry)) + errs++ + } + if uint8(dec.entry>>8) != uint8(sym) { + fmt.Fprintf(w, "symbol 0x%x decoder output mismatch (enc: %d, dec:%d).\n", sym, sym, uint8(dec.entry>>8)) + errs++ + } + if errs > 0 { + fmt.Fprintf(w, "%d errros in base, stopping\n", errs) + continue + } + // Ensure that all combinations are covered. + for i := uint16(0); i < (1 << ub); i++ { + vval := top | i + dec := dt[vval] + if uint8(dec.entry) != enc.nBits { + fmt.Fprintf(w, "symbol 0x%x bit size mismatch (enc: %d, dec:%d).\n", vval, enc.nBits, uint8(dec.entry)) + errs++ + } + if uint8(dec.entry>>8) != uint8(sym) { + fmt.Fprintf(w, "symbol 0x%x decoder output mismatch (enc: %d, dec:%d).\n", vval, sym, uint8(dec.entry>>8)) + errs++ + } + if errs > 20 { + fmt.Fprintf(w, "%d errros, stopping\n", errs) + break + } + } + if errs == 0 { + ok++ + broken-- + } + } + if broken > 0 { + fmt.Fprintf(w, "%d broken, %d ok\n", broken, ok) + } +} diff --git a/vendor/github.com/klauspost/compress/huff0/huff0.go b/vendor/github.com/klauspost/compress/huff0/huff0.go new file mode 100644 index 000000000..6bc23bbf0 --- /dev/null +++ b/vendor/github.com/klauspost/compress/huff0/huff0.go @@ -0,0 +1,258 @@ +// Package huff0 provides fast huffman encoding as used in zstd. +// +// See README.md at https://github.com/klauspost/compress/tree/master/huff0 for details. +package huff0 + +import ( + "errors" + "fmt" + "math" + "math/bits" + + "github.com/klauspost/compress/fse" +) + +const ( + maxSymbolValue = 255 + + // zstandard limits tablelog to 11, see: + // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#huffman-tree-description + tableLogMax = 11 + tableLogDefault = 11 + minTablelog = 5 + huffNodesLen = 512 + + // BlockSizeMax is maximum input size for a single block uncompressed. + BlockSizeMax = 1<<18 - 1 +) + +var ( + // ErrIncompressible is returned when input is judged to be too hard to compress. + ErrIncompressible = errors.New("input is not compressible") + + // ErrUseRLE is returned from the compressor when the input is a single byte value repeated. + ErrUseRLE = errors.New("input is single value repeated") + + // ErrTooBig is return if input is too large for a single block. + ErrTooBig = errors.New("input too big") + + // ErrMaxDecodedSizeExceeded is return if input is too large for a single block. + ErrMaxDecodedSizeExceeded = errors.New("maximum output size exceeded") +) + +type ReusePolicy uint8 + +const ( + // ReusePolicyAllow will allow reuse if it produces smaller output. + ReusePolicyAllow ReusePolicy = iota + + // ReusePolicyPrefer will re-use aggressively if possible. + // This will not check if a new table will produce smaller output, + // except if the current table is impossible to use or + // compressed output is bigger than input. + ReusePolicyPrefer + + // ReusePolicyNone will disable re-use of tables. + // This is slightly faster than ReusePolicyAllow but may produce larger output. + ReusePolicyNone +) + +type Scratch struct { + count [maxSymbolValue + 1]uint32 + + // Per block parameters. + // These can be used to override compression parameters of the block. + // Do not touch, unless you know what you are doing. + + // Out is output buffer. + // If the scratch is re-used before the caller is done processing the output, + // set this field to nil. + // Otherwise the output buffer will be re-used for next Compression/Decompression step + // and allocation will be avoided. + Out []byte + + // OutTable will contain the table data only, if a new table has been generated. + // Slice of the returned data. + OutTable []byte + + // OutData will contain the compressed data. + // Slice of the returned data. + OutData []byte + + // MaxSymbolValue will override the maximum symbol value of the next block. + MaxSymbolValue uint8 + + // TableLog will attempt to override the tablelog for the next block. + // Must be <= 11. + TableLog uint8 + + // Reuse will specify the reuse policy + Reuse ReusePolicy + + // WantLogLess allows to specify a log 2 reduction that should at least be achieved, + // otherwise the block will be returned as incompressible. + // The reduction should then at least be (input size >> WantLogLess) + // If WantLogLess == 0 any improvement will do. + WantLogLess uint8 + + // MaxDecodedSize will set the maximum allowed output size. + // This value will automatically be set to BlockSizeMax if not set. + // Decoders will return ErrMaxDecodedSizeExceeded is this limit is exceeded. + MaxDecodedSize int + + br byteReader + symbolLen uint16 // Length of active part of the symbol table. + maxCount int // count of the most probable symbol + clearCount bool // clear count + actualTableLog uint8 // Selected tablelog. + prevTable cTable // Table used for previous compression. + cTable cTable // compression table + dt dTable // decompression table + nodes []nodeElt + tmpOut [4][]byte + fse *fse.Scratch + huffWeight [maxSymbolValue + 1]byte +} + +func (s *Scratch) prepare(in []byte) (*Scratch, error) { + if len(in) > BlockSizeMax { + return nil, ErrTooBig + } + if s == nil { + s = &Scratch{} + } + if s.MaxSymbolValue == 0 { + s.MaxSymbolValue = maxSymbolValue + } + if s.TableLog == 0 { + s.TableLog = tableLogDefault + } + if s.TableLog > tableLogMax { + return nil, fmt.Errorf("tableLog (%d) > maxTableLog (%d)", s.TableLog, tableLogMax) + } + if s.MaxDecodedSize <= 0 || s.MaxDecodedSize > BlockSizeMax { + s.MaxDecodedSize = BlockSizeMax + } + if s.clearCount && s.maxCount == 0 { + for i := range s.count { + s.count[i] = 0 + } + s.clearCount = false + } + if cap(s.Out) == 0 { + s.Out = make([]byte, 0, len(in)) + } + s.Out = s.Out[:0] + + s.OutTable = nil + s.OutData = nil + if cap(s.nodes) < huffNodesLen+1 { + s.nodes = make([]nodeElt, 0, huffNodesLen+1) + } + s.nodes = s.nodes[:0] + if s.fse == nil { + s.fse = &fse.Scratch{} + } + s.br.init(in) + + return s, nil +} + +type cTable []cTableEntry + +func (c cTable) write(s *Scratch) error { + var ( + // precomputed conversion table + bitsToWeight [tableLogMax + 1]byte + huffLog = s.actualTableLog + // last weight is not saved. + maxSymbolValue = uint8(s.symbolLen - 1) + huffWeight = s.huffWeight[:256] + ) + const ( + maxFSETableLog = 6 + ) + // convert to weight + bitsToWeight[0] = 0 + for n := uint8(1); n < huffLog+1; n++ { + bitsToWeight[n] = huffLog + 1 - n + } + + // Acquire histogram for FSE. + hist := s.fse.Histogram() + hist = hist[:256] + for i := range hist[:16] { + hist[i] = 0 + } + for n := uint8(0); n < maxSymbolValue; n++ { + v := bitsToWeight[c[n].nBits] & 15 + huffWeight[n] = v + hist[v]++ + } + + // FSE compress if feasible. + if maxSymbolValue >= 2 { + huffMaxCnt := uint32(0) + huffMax := uint8(0) + for i, v := range hist[:16] { + if v == 0 { + continue + } + huffMax = byte(i) + if v > huffMaxCnt { + huffMaxCnt = v + } + } + s.fse.HistogramFinished(huffMax, int(huffMaxCnt)) + s.fse.TableLog = maxFSETableLog + b, err := fse.Compress(huffWeight[:maxSymbolValue], s.fse) + if err == nil && len(b) < int(s.symbolLen>>1) { + s.Out = append(s.Out, uint8(len(b))) + s.Out = append(s.Out, b...) + return nil + } + // Unable to compress (RLE/uncompressible) + } + // write raw values as 4-bits (max : 15) + if maxSymbolValue > (256 - 128) { + // should not happen : likely means source cannot be compressed + return ErrIncompressible + } + op := s.Out + // special case, pack weights 4 bits/weight. + op = append(op, 128|(maxSymbolValue-1)) + // be sure it doesn't cause msan issue in final combination + huffWeight[maxSymbolValue] = 0 + for n := uint16(0); n < uint16(maxSymbolValue); n += 2 { + op = append(op, (huffWeight[n]<<4)|huffWeight[n+1]) + } + s.Out = op + return nil +} + +// estimateSize returns the estimated size in bytes of the input represented in the +// histogram supplied. +func (c cTable) estimateSize(hist []uint32) int { + nbBits := uint32(7) + for i, v := range c[:len(hist)] { + nbBits += uint32(v.nBits) * hist[i] + } + return int(nbBits >> 3) +} + +// minSize returns the minimum possible size considering the shannon limit. +func (s *Scratch) minSize(total int) int { + nbBits := float64(7) + fTotal := float64(total) + for _, v := range s.count[:s.symbolLen] { + n := float64(v) + if n > 0 { + nbBits += math.Log2(fTotal/n) * n + } + } + return int(nbBits) >> 3 +} + +func highBit32(val uint32) (n uint32) { + return uint32(bits.Len32(val) - 1) +} diff --git a/vendor/github.com/klauspost/compress/snappy/.gitignore b/vendor/github.com/klauspost/compress/snappy/.gitignore new file mode 100644 index 000000000..042091d9b --- /dev/null +++ b/vendor/github.com/klauspost/compress/snappy/.gitignore @@ -0,0 +1,16 @@ +cmd/snappytool/snappytool +testdata/bench + +# These explicitly listed benchmark data files are for an obsolete version of +# snappy_test.go. +testdata/alice29.txt +testdata/asyoulik.txt +testdata/fireworks.jpeg +testdata/geo.protodata +testdata/html +testdata/html_x_4 +testdata/kppkn.gtb +testdata/lcet10.txt +testdata/paper-100k.pdf +testdata/plrabn12.txt +testdata/urls.10K diff --git a/vendor/github.com/klauspost/compress/snappy/AUTHORS b/vendor/github.com/klauspost/compress/snappy/AUTHORS new file mode 100644 index 000000000..bcfa19520 --- /dev/null +++ b/vendor/github.com/klauspost/compress/snappy/AUTHORS @@ -0,0 +1,15 @@ +# This is the official list of Snappy-Go authors for copyright purposes. +# This file is distinct from the CONTRIBUTORS files. +# See the latter for an explanation. + +# Names should be added to this file as +# Name or Organization +# The email address is not required for organizations. + +# Please keep the list sorted. + +Damian Gryski +Google Inc. +Jan Mercl <0xjnml@gmail.com> +Rodolfo Carvalho +Sebastien Binet diff --git a/vendor/github.com/klauspost/compress/snappy/CONTRIBUTORS b/vendor/github.com/klauspost/compress/snappy/CONTRIBUTORS new file mode 100644 index 000000000..931ae3160 --- /dev/null +++ b/vendor/github.com/klauspost/compress/snappy/CONTRIBUTORS @@ -0,0 +1,37 @@ +# This is the official list of people who can contribute +# (and typically have contributed) code to the Snappy-Go repository. +# The AUTHORS file lists the copyright holders; this file +# lists people. For example, Google employees are listed here +# but not in AUTHORS, because Google holds the copyright. +# +# The submission process automatically checks to make sure +# that people submitting code are listed in this file (by email address). +# +# Names should be added to this file only after verifying that +# the individual or the individual's organization has agreed to +# the appropriate Contributor License Agreement, found here: +# +# http://code.google.com/legal/individual-cla-v1.0.html +# http://code.google.com/legal/corporate-cla-v1.0.html +# +# The agreement for individuals can be filled out on the web. +# +# When adding J Random Contributor's name to this file, +# either J's name or J's organization's name should be +# added to the AUTHORS file, depending on whether the +# individual or corporate CLA was used. + +# Names should be added to this file like so: +# Name + +# Please keep the list sorted. + +Damian Gryski +Jan Mercl <0xjnml@gmail.com> +Kai Backman +Marc-Antoine Ruel +Nigel Tao +Rob Pike +Rodolfo Carvalho +Russ Cox +Sebastien Binet diff --git a/vendor/github.com/klauspost/compress/snappy/LICENSE b/vendor/github.com/klauspost/compress/snappy/LICENSE new file mode 100644 index 000000000..6050c10f4 --- /dev/null +++ b/vendor/github.com/klauspost/compress/snappy/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/klauspost/compress/snappy/README b/vendor/github.com/klauspost/compress/snappy/README new file mode 100644 index 000000000..cea12879a --- /dev/null +++ b/vendor/github.com/klauspost/compress/snappy/README @@ -0,0 +1,107 @@ +The Snappy compression format in the Go programming language. + +To download and install from source: +$ go get github.com/golang/snappy + +Unless otherwise noted, the Snappy-Go source files are distributed +under the BSD-style license found in the LICENSE file. + + + +Benchmarks. + +The golang/snappy benchmarks include compressing (Z) and decompressing (U) ten +or so files, the same set used by the C++ Snappy code (github.com/google/snappy +and note the "google", not "golang"). On an "Intel(R) Core(TM) i7-3770 CPU @ +3.40GHz", Go's GOARCH=amd64 numbers as of 2016-05-29: + +"go test -test.bench=." + +_UFlat0-8 2.19GB/s ± 0% html +_UFlat1-8 1.41GB/s ± 0% urls +_UFlat2-8 23.5GB/s ± 2% jpg +_UFlat3-8 1.91GB/s ± 0% jpg_200 +_UFlat4-8 14.0GB/s ± 1% pdf +_UFlat5-8 1.97GB/s ± 0% html4 +_UFlat6-8 814MB/s ± 0% txt1 +_UFlat7-8 785MB/s ± 0% txt2 +_UFlat8-8 857MB/s ± 0% txt3 +_UFlat9-8 719MB/s ± 1% txt4 +_UFlat10-8 2.84GB/s ± 0% pb +_UFlat11-8 1.05GB/s ± 0% gaviota + +_ZFlat0-8 1.04GB/s ± 0% html +_ZFlat1-8 534MB/s ± 0% urls +_ZFlat2-8 15.7GB/s ± 1% jpg +_ZFlat3-8 740MB/s ± 3% jpg_200 +_ZFlat4-8 9.20GB/s ± 1% pdf +_ZFlat5-8 991MB/s ± 0% html4 +_ZFlat6-8 379MB/s ± 0% txt1 +_ZFlat7-8 352MB/s ± 0% txt2 +_ZFlat8-8 396MB/s ± 1% txt3 +_ZFlat9-8 327MB/s ± 1% txt4 +_ZFlat10-8 1.33GB/s ± 1% pb +_ZFlat11-8 605MB/s ± 1% gaviota + + + +"go test -test.bench=. -tags=noasm" + +_UFlat0-8 621MB/s ± 2% html +_UFlat1-8 494MB/s ± 1% urls +_UFlat2-8 23.2GB/s ± 1% jpg +_UFlat3-8 1.12GB/s ± 1% jpg_200 +_UFlat4-8 4.35GB/s ± 1% pdf +_UFlat5-8 609MB/s ± 0% html4 +_UFlat6-8 296MB/s ± 0% txt1 +_UFlat7-8 288MB/s ± 0% txt2 +_UFlat8-8 309MB/s ± 1% txt3 +_UFlat9-8 280MB/s ± 1% txt4 +_UFlat10-8 753MB/s ± 0% pb +_UFlat11-8 400MB/s ± 0% gaviota + +_ZFlat0-8 409MB/s ± 1% html +_ZFlat1-8 250MB/s ± 1% urls +_ZFlat2-8 12.3GB/s ± 1% jpg +_ZFlat3-8 132MB/s ± 0% jpg_200 +_ZFlat4-8 2.92GB/s ± 0% pdf +_ZFlat5-8 405MB/s ± 1% html4 +_ZFlat6-8 179MB/s ± 1% txt1 +_ZFlat7-8 170MB/s ± 1% txt2 +_ZFlat8-8 189MB/s ± 1% txt3 +_ZFlat9-8 164MB/s ± 1% txt4 +_ZFlat10-8 479MB/s ± 1% pb +_ZFlat11-8 270MB/s ± 1% gaviota + + + +For comparison (Go's encoded output is byte-for-byte identical to C++'s), here +are the numbers from C++ Snappy's + +make CXXFLAGS="-O2 -DNDEBUG -g" clean snappy_unittest.log && cat snappy_unittest.log + +BM_UFlat/0 2.4GB/s html +BM_UFlat/1 1.4GB/s urls +BM_UFlat/2 21.8GB/s jpg +BM_UFlat/3 1.5GB/s jpg_200 +BM_UFlat/4 13.3GB/s pdf +BM_UFlat/5 2.1GB/s html4 +BM_UFlat/6 1.0GB/s txt1 +BM_UFlat/7 959.4MB/s txt2 +BM_UFlat/8 1.0GB/s txt3 +BM_UFlat/9 864.5MB/s txt4 +BM_UFlat/10 2.9GB/s pb +BM_UFlat/11 1.2GB/s gaviota + +BM_ZFlat/0 944.3MB/s html (22.31 %) +BM_ZFlat/1 501.6MB/s urls (47.78 %) +BM_ZFlat/2 14.3GB/s jpg (99.95 %) +BM_ZFlat/3 538.3MB/s jpg_200 (73.00 %) +BM_ZFlat/4 8.3GB/s pdf (83.30 %) +BM_ZFlat/5 903.5MB/s html4 (22.52 %) +BM_ZFlat/6 336.0MB/s txt1 (57.88 %) +BM_ZFlat/7 312.3MB/s txt2 (61.91 %) +BM_ZFlat/8 353.1MB/s txt3 (54.99 %) +BM_ZFlat/9 289.9MB/s txt4 (66.26 %) +BM_ZFlat/10 1.2GB/s pb (19.68 %) +BM_ZFlat/11 527.4MB/s gaviota (37.72 %) diff --git a/vendor/github.com/klauspost/compress/snappy/decode.go b/vendor/github.com/klauspost/compress/snappy/decode.go new file mode 100644 index 000000000..72efb0353 --- /dev/null +++ b/vendor/github.com/klauspost/compress/snappy/decode.go @@ -0,0 +1,237 @@ +// Copyright 2011 The Snappy-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package snappy + +import ( + "encoding/binary" + "errors" + "io" +) + +var ( + // ErrCorrupt reports that the input is invalid. + ErrCorrupt = errors.New("snappy: corrupt input") + // ErrTooLarge reports that the uncompressed length is too large. + ErrTooLarge = errors.New("snappy: decoded block is too large") + // ErrUnsupported reports that the input isn't supported. + ErrUnsupported = errors.New("snappy: unsupported input") + + errUnsupportedLiteralLength = errors.New("snappy: unsupported literal length") +) + +// DecodedLen returns the length of the decoded block. +func DecodedLen(src []byte) (int, error) { + v, _, err := decodedLen(src) + return v, err +} + +// decodedLen returns the length of the decoded block and the number of bytes +// that the length header occupied. +func decodedLen(src []byte) (blockLen, headerLen int, err error) { + v, n := binary.Uvarint(src) + if n <= 0 || v > 0xffffffff { + return 0, 0, ErrCorrupt + } + + const wordSize = 32 << (^uint(0) >> 32 & 1) + if wordSize == 32 && v > 0x7fffffff { + return 0, 0, ErrTooLarge + } + return int(v), n, nil +} + +const ( + decodeErrCodeCorrupt = 1 + decodeErrCodeUnsupportedLiteralLength = 2 +) + +// Decode returns the decoded form of src. The returned slice may be a sub- +// slice of dst if dst was large enough to hold the entire decoded block. +// Otherwise, a newly allocated slice will be returned. +// +// The dst and src must not overlap. It is valid to pass a nil dst. +func Decode(dst, src []byte) ([]byte, error) { + dLen, s, err := decodedLen(src) + if err != nil { + return nil, err + } + if dLen <= len(dst) { + dst = dst[:dLen] + } else { + dst = make([]byte, dLen) + } + switch decode(dst, src[s:]) { + case 0: + return dst, nil + case decodeErrCodeUnsupportedLiteralLength: + return nil, errUnsupportedLiteralLength + } + return nil, ErrCorrupt +} + +// NewReader returns a new Reader that decompresses from r, using the framing +// format described at +// https://github.com/google/snappy/blob/master/framing_format.txt +func NewReader(r io.Reader) *Reader { + return &Reader{ + r: r, + decoded: make([]byte, maxBlockSize), + buf: make([]byte, maxEncodedLenOfMaxBlockSize+checksumSize), + } +} + +// Reader is an io.Reader that can read Snappy-compressed bytes. +type Reader struct { + r io.Reader + err error + decoded []byte + buf []byte + // decoded[i:j] contains decoded bytes that have not yet been passed on. + i, j int + readHeader bool +} + +// Reset discards any buffered data, resets all state, and switches the Snappy +// reader to read from r. This permits reusing a Reader rather than allocating +// a new one. +func (r *Reader) Reset(reader io.Reader) { + r.r = reader + r.err = nil + r.i = 0 + r.j = 0 + r.readHeader = false +} + +func (r *Reader) readFull(p []byte, allowEOF bool) (ok bool) { + if _, r.err = io.ReadFull(r.r, p); r.err != nil { + if r.err == io.ErrUnexpectedEOF || (r.err == io.EOF && !allowEOF) { + r.err = ErrCorrupt + } + return false + } + return true +} + +// Read satisfies the io.Reader interface. +func (r *Reader) Read(p []byte) (int, error) { + if r.err != nil { + return 0, r.err + } + for { + if r.i < r.j { + n := copy(p, r.decoded[r.i:r.j]) + r.i += n + return n, nil + } + if !r.readFull(r.buf[:4], true) { + return 0, r.err + } + chunkType := r.buf[0] + if !r.readHeader { + if chunkType != chunkTypeStreamIdentifier { + r.err = ErrCorrupt + return 0, r.err + } + r.readHeader = true + } + chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16 + if chunkLen > len(r.buf) { + r.err = ErrUnsupported + return 0, r.err + } + + // The chunk types are specified at + // https://github.com/google/snappy/blob/master/framing_format.txt + switch chunkType { + case chunkTypeCompressedData: + // Section 4.2. Compressed data (chunk type 0x00). + if chunkLen < checksumSize { + r.err = ErrCorrupt + return 0, r.err + } + buf := r.buf[:chunkLen] + if !r.readFull(buf, false) { + return 0, r.err + } + checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 + buf = buf[checksumSize:] + + n, err := DecodedLen(buf) + if err != nil { + r.err = err + return 0, r.err + } + if n > len(r.decoded) { + r.err = ErrCorrupt + return 0, r.err + } + if _, err := Decode(r.decoded, buf); err != nil { + r.err = err + return 0, r.err + } + if crc(r.decoded[:n]) != checksum { + r.err = ErrCorrupt + return 0, r.err + } + r.i, r.j = 0, n + continue + + case chunkTypeUncompressedData: + // Section 4.3. Uncompressed data (chunk type 0x01). + if chunkLen < checksumSize { + r.err = ErrCorrupt + return 0, r.err + } + buf := r.buf[:checksumSize] + if !r.readFull(buf, false) { + return 0, r.err + } + checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 + // Read directly into r.decoded instead of via r.buf. + n := chunkLen - checksumSize + if n > len(r.decoded) { + r.err = ErrCorrupt + return 0, r.err + } + if !r.readFull(r.decoded[:n], false) { + return 0, r.err + } + if crc(r.decoded[:n]) != checksum { + r.err = ErrCorrupt + return 0, r.err + } + r.i, r.j = 0, n + continue + + case chunkTypeStreamIdentifier: + // Section 4.1. Stream identifier (chunk type 0xff). + if chunkLen != len(magicBody) { + r.err = ErrCorrupt + return 0, r.err + } + if !r.readFull(r.buf[:len(magicBody)], false) { + return 0, r.err + } + for i := 0; i < len(magicBody); i++ { + if r.buf[i] != magicBody[i] { + r.err = ErrCorrupt + return 0, r.err + } + } + continue + } + + if chunkType <= 0x7f { + // Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f). + r.err = ErrUnsupported + return 0, r.err + } + // Section 4.4 Padding (chunk type 0xfe). + // Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd). + if !r.readFull(r.buf[:chunkLen], false) { + return 0, r.err + } + } +} diff --git a/vendor/github.com/klauspost/compress/snappy/decode_amd64.go b/vendor/github.com/klauspost/compress/snappy/decode_amd64.go new file mode 100644 index 000000000..fcd192b84 --- /dev/null +++ b/vendor/github.com/klauspost/compress/snappy/decode_amd64.go @@ -0,0 +1,14 @@ +// Copyright 2016 The Snappy-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !appengine +// +build gc +// +build !noasm + +package snappy + +// decode has the same semantics as in decode_other.go. +// +//go:noescape +func decode(dst, src []byte) int diff --git a/vendor/github.com/klauspost/compress/snappy/decode_amd64.s b/vendor/github.com/klauspost/compress/snappy/decode_amd64.s new file mode 100644 index 000000000..1c66e3723 --- /dev/null +++ b/vendor/github.com/klauspost/compress/snappy/decode_amd64.s @@ -0,0 +1,482 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !appengine +// +build gc +// +build !noasm + +#include "textflag.h" + +// The asm code generally follows the pure Go code in decode_other.go, except +// where marked with a "!!!". + +// func decode(dst, src []byte) int +// +// All local variables fit into registers. The non-zero stack size is only to +// spill registers and push args when issuing a CALL. The register allocation: +// - AX scratch +// - BX scratch +// - CX length or x +// - DX offset +// - SI &src[s] +// - DI &dst[d] +// + R8 dst_base +// + R9 dst_len +// + R10 dst_base + dst_len +// + R11 src_base +// + R12 src_len +// + R13 src_base + src_len +// - R14 used by doCopy +// - R15 used by doCopy +// +// The registers R8-R13 (marked with a "+") are set at the start of the +// function, and after a CALL returns, and are not otherwise modified. +// +// The d variable is implicitly DI - R8, and len(dst)-d is R10 - DI. +// The s variable is implicitly SI - R11, and len(src)-s is R13 - SI. +TEXT ·decode(SB), NOSPLIT, $48-56 + // Initialize SI, DI and R8-R13. + MOVQ dst_base+0(FP), R8 + MOVQ dst_len+8(FP), R9 + MOVQ R8, DI + MOVQ R8, R10 + ADDQ R9, R10 + MOVQ src_base+24(FP), R11 + MOVQ src_len+32(FP), R12 + MOVQ R11, SI + MOVQ R11, R13 + ADDQ R12, R13 + +loop: + // for s < len(src) + CMPQ SI, R13 + JEQ end + + // CX = uint32(src[s]) + // + // switch src[s] & 0x03 + MOVBLZX (SI), CX + MOVL CX, BX + ANDL $3, BX + CMPL BX, $1 + JAE tagCopy + + // ---------------------------------------- + // The code below handles literal tags. + + // case tagLiteral: + // x := uint32(src[s] >> 2) + // switch + SHRL $2, CX + CMPL CX, $60 + JAE tagLit60Plus + + // case x < 60: + // s++ + INCQ SI + +doLit: + // This is the end of the inner "switch", when we have a literal tag. + // + // We assume that CX == x and x fits in a uint32, where x is the variable + // used in the pure Go decode_other.go code. + + // length = int(x) + 1 + // + // Unlike the pure Go code, we don't need to check if length <= 0 because + // CX can hold 64 bits, so the increment cannot overflow. + INCQ CX + + // Prepare to check if copying length bytes will run past the end of dst or + // src. + // + // AX = len(dst) - d + // BX = len(src) - s + MOVQ R10, AX + SUBQ DI, AX + MOVQ R13, BX + SUBQ SI, BX + + // !!! Try a faster technique for short (16 or fewer bytes) copies. + // + // if length > 16 || len(dst)-d < 16 || len(src)-s < 16 { + // goto callMemmove // Fall back on calling runtime·memmove. + // } + // + // The C++ snappy code calls this TryFastAppend. It also checks len(src)-s + // against 21 instead of 16, because it cannot assume that all of its input + // is contiguous in memory and so it needs to leave enough source bytes to + // read the next tag without refilling buffers, but Go's Decode assumes + // contiguousness (the src argument is a []byte). + CMPQ CX, $16 + JGT callMemmove + CMPQ AX, $16 + JLT callMemmove + CMPQ BX, $16 + JLT callMemmove + + // !!! Implement the copy from src to dst as a 16-byte load and store. + // (Decode's documentation says that dst and src must not overlap.) + // + // This always copies 16 bytes, instead of only length bytes, but that's + // OK. If the input is a valid Snappy encoding then subsequent iterations + // will fix up the overrun. Otherwise, Decode returns a nil []byte (and a + // non-nil error), so the overrun will be ignored. + // + // Note that on amd64, it is legal and cheap to issue unaligned 8-byte or + // 16-byte loads and stores. This technique probably wouldn't be as + // effective on architectures that are fussier about alignment. + MOVOU 0(SI), X0 + MOVOU X0, 0(DI) + + // d += length + // s += length + ADDQ CX, DI + ADDQ CX, SI + JMP loop + +callMemmove: + // if length > len(dst)-d || length > len(src)-s { etc } + CMPQ CX, AX + JGT errCorrupt + CMPQ CX, BX + JGT errCorrupt + + // copy(dst[d:], src[s:s+length]) + // + // This means calling runtime·memmove(&dst[d], &src[s], length), so we push + // DI, SI and CX as arguments. Coincidentally, we also need to spill those + // three registers to the stack, to save local variables across the CALL. + MOVQ DI, 0(SP) + MOVQ SI, 8(SP) + MOVQ CX, 16(SP) + MOVQ DI, 24(SP) + MOVQ SI, 32(SP) + MOVQ CX, 40(SP) + CALL runtime·memmove(SB) + + // Restore local variables: unspill registers from the stack and + // re-calculate R8-R13. + MOVQ 24(SP), DI + MOVQ 32(SP), SI + MOVQ 40(SP), CX + MOVQ dst_base+0(FP), R8 + MOVQ dst_len+8(FP), R9 + MOVQ R8, R10 + ADDQ R9, R10 + MOVQ src_base+24(FP), R11 + MOVQ src_len+32(FP), R12 + MOVQ R11, R13 + ADDQ R12, R13 + + // d += length + // s += length + ADDQ CX, DI + ADDQ CX, SI + JMP loop + +tagLit60Plus: + // !!! This fragment does the + // + // s += x - 58; if uint(s) > uint(len(src)) { etc } + // + // checks. In the asm version, we code it once instead of once per switch case. + ADDQ CX, SI + SUBQ $58, SI + CMPQ SI, R13 + JA errCorrupt + + // case x == 60: + CMPL CX, $61 + JEQ tagLit61 + JA tagLit62Plus + + // x = uint32(src[s-1]) + MOVBLZX -1(SI), CX + JMP doLit + +tagLit61: + // case x == 61: + // x = uint32(src[s-2]) | uint32(src[s-1])<<8 + MOVWLZX -2(SI), CX + JMP doLit + +tagLit62Plus: + CMPL CX, $62 + JA tagLit63 + + // case x == 62: + // x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16 + MOVWLZX -3(SI), CX + MOVBLZX -1(SI), BX + SHLL $16, BX + ORL BX, CX + JMP doLit + +tagLit63: + // case x == 63: + // x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24 + MOVL -4(SI), CX + JMP doLit + +// The code above handles literal tags. +// ---------------------------------------- +// The code below handles copy tags. + +tagCopy4: + // case tagCopy4: + // s += 5 + ADDQ $5, SI + + // if uint(s) > uint(len(src)) { etc } + CMPQ SI, R13 + JA errCorrupt + + // length = 1 + int(src[s-5])>>2 + SHRQ $2, CX + INCQ CX + + // offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24) + MOVLQZX -4(SI), DX + JMP doCopy + +tagCopy2: + // case tagCopy2: + // s += 3 + ADDQ $3, SI + + // if uint(s) > uint(len(src)) { etc } + CMPQ SI, R13 + JA errCorrupt + + // length = 1 + int(src[s-3])>>2 + SHRQ $2, CX + INCQ CX + + // offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8) + MOVWQZX -2(SI), DX + JMP doCopy + +tagCopy: + // We have a copy tag. We assume that: + // - BX == src[s] & 0x03 + // - CX == src[s] + CMPQ BX, $2 + JEQ tagCopy2 + JA tagCopy4 + + // case tagCopy1: + // s += 2 + ADDQ $2, SI + + // if uint(s) > uint(len(src)) { etc } + CMPQ SI, R13 + JA errCorrupt + + // offset = int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1])) + MOVQ CX, DX + ANDQ $0xe0, DX + SHLQ $3, DX + MOVBQZX -1(SI), BX + ORQ BX, DX + + // length = 4 + int(src[s-2])>>2&0x7 + SHRQ $2, CX + ANDQ $7, CX + ADDQ $4, CX + +doCopy: + // This is the end of the outer "switch", when we have a copy tag. + // + // We assume that: + // - CX == length && CX > 0 + // - DX == offset + + // if offset <= 0 { etc } + CMPQ DX, $0 + JLE errCorrupt + + // if d < offset { etc } + MOVQ DI, BX + SUBQ R8, BX + CMPQ BX, DX + JLT errCorrupt + + // if length > len(dst)-d { etc } + MOVQ R10, BX + SUBQ DI, BX + CMPQ CX, BX + JGT errCorrupt + + // forwardCopy(dst[d:d+length], dst[d-offset:]); d += length + // + // Set: + // - R14 = len(dst)-d + // - R15 = &dst[d-offset] + MOVQ R10, R14 + SUBQ DI, R14 + MOVQ DI, R15 + SUBQ DX, R15 + + // !!! Try a faster technique for short (16 or fewer bytes) forward copies. + // + // First, try using two 8-byte load/stores, similar to the doLit technique + // above. Even if dst[d:d+length] and dst[d-offset:] can overlap, this is + // still OK if offset >= 8. Note that this has to be two 8-byte load/stores + // and not one 16-byte load/store, and the first store has to be before the + // second load, due to the overlap if offset is in the range [8, 16). + // + // if length > 16 || offset < 8 || len(dst)-d < 16 { + // goto slowForwardCopy + // } + // copy 16 bytes + // d += length + CMPQ CX, $16 + JGT slowForwardCopy + CMPQ DX, $8 + JLT slowForwardCopy + CMPQ R14, $16 + JLT slowForwardCopy + MOVQ 0(R15), AX + MOVQ AX, 0(DI) + MOVQ 8(R15), BX + MOVQ BX, 8(DI) + ADDQ CX, DI + JMP loop + +slowForwardCopy: + // !!! If the forward copy is longer than 16 bytes, or if offset < 8, we + // can still try 8-byte load stores, provided we can overrun up to 10 extra + // bytes. As above, the overrun will be fixed up by subsequent iterations + // of the outermost loop. + // + // The C++ snappy code calls this technique IncrementalCopyFastPath. Its + // commentary says: + // + // ---- + // + // The main part of this loop is a simple copy of eight bytes at a time + // until we've copied (at least) the requested amount of bytes. However, + // if d and d-offset are less than eight bytes apart (indicating a + // repeating pattern of length < 8), we first need to expand the pattern in + // order to get the correct results. For instance, if the buffer looks like + // this, with the eight-byte and patterns marked as + // intervals: + // + // abxxxxxxxxxxxx + // [------] d-offset + // [------] d + // + // a single eight-byte copy from to will repeat the pattern + // once, after which we can move two bytes without moving : + // + // ababxxxxxxxxxx + // [------] d-offset + // [------] d + // + // and repeat the exercise until the two no longer overlap. + // + // This allows us to do very well in the special case of one single byte + // repeated many times, without taking a big hit for more general cases. + // + // The worst case of extra writing past the end of the match occurs when + // offset == 1 and length == 1; the last copy will read from byte positions + // [0..7] and write to [4..11], whereas it was only supposed to write to + // position 1. Thus, ten excess bytes. + // + // ---- + // + // That "10 byte overrun" worst case is confirmed by Go's + // TestSlowForwardCopyOverrun, which also tests the fixUpSlowForwardCopy + // and finishSlowForwardCopy algorithm. + // + // if length > len(dst)-d-10 { + // goto verySlowForwardCopy + // } + SUBQ $10, R14 + CMPQ CX, R14 + JGT verySlowForwardCopy + +makeOffsetAtLeast8: + // !!! As above, expand the pattern so that offset >= 8 and we can use + // 8-byte load/stores. + // + // for offset < 8 { + // copy 8 bytes from dst[d-offset:] to dst[d:] + // length -= offset + // d += offset + // offset += offset + // // The two previous lines together means that d-offset, and therefore + // // R15, is unchanged. + // } + CMPQ DX, $8 + JGE fixUpSlowForwardCopy + MOVQ (R15), BX + MOVQ BX, (DI) + SUBQ DX, CX + ADDQ DX, DI + ADDQ DX, DX + JMP makeOffsetAtLeast8 + +fixUpSlowForwardCopy: + // !!! Add length (which might be negative now) to d (implied by DI being + // &dst[d]) so that d ends up at the right place when we jump back to the + // top of the loop. Before we do that, though, we save DI to AX so that, if + // length is positive, copying the remaining length bytes will write to the + // right place. + MOVQ DI, AX + ADDQ CX, DI + +finishSlowForwardCopy: + // !!! Repeat 8-byte load/stores until length <= 0. Ending with a negative + // length means that we overrun, but as above, that will be fixed up by + // subsequent iterations of the outermost loop. + CMPQ CX, $0 + JLE loop + MOVQ (R15), BX + MOVQ BX, (AX) + ADDQ $8, R15 + ADDQ $8, AX + SUBQ $8, CX + JMP finishSlowForwardCopy + +verySlowForwardCopy: + // verySlowForwardCopy is a simple implementation of forward copy. In C + // parlance, this is a do/while loop instead of a while loop, since we know + // that length > 0. In Go syntax: + // + // for { + // dst[d] = dst[d - offset] + // d++ + // length-- + // if length == 0 { + // break + // } + // } + MOVB (R15), BX + MOVB BX, (DI) + INCQ R15 + INCQ DI + DECQ CX + JNZ verySlowForwardCopy + JMP loop + +// The code above handles copy tags. +// ---------------------------------------- + +end: + // This is the end of the "for s < len(src)". + // + // if d != len(dst) { etc } + CMPQ DI, R10 + JNE errCorrupt + + // return 0 + MOVQ $0, ret+48(FP) + RET + +errCorrupt: + // return decodeErrCodeCorrupt + MOVQ $1, ret+48(FP) + RET diff --git a/vendor/github.com/klauspost/compress/snappy/decode_other.go b/vendor/github.com/klauspost/compress/snappy/decode_other.go new file mode 100644 index 000000000..94a96c5d7 --- /dev/null +++ b/vendor/github.com/klauspost/compress/snappy/decode_other.go @@ -0,0 +1,115 @@ +// Copyright 2016 The Snappy-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !amd64 appengine !gc noasm + +package snappy + +// decode writes the decoding of src to dst. It assumes that the varint-encoded +// length of the decompressed bytes has already been read, and that len(dst) +// equals that length. +// +// It returns 0 on success or a decodeErrCodeXxx error code on failure. +func decode(dst, src []byte) int { + var d, s, offset, length int + for s < len(src) { + switch src[s] & 0x03 { + case tagLiteral: + x := uint32(src[s] >> 2) + switch { + case x < 60: + s++ + case x == 60: + s += 2 + if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. + return decodeErrCodeCorrupt + } + x = uint32(src[s-1]) + case x == 61: + s += 3 + if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. + return decodeErrCodeCorrupt + } + x = uint32(src[s-2]) | uint32(src[s-1])<<8 + case x == 62: + s += 4 + if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. + return decodeErrCodeCorrupt + } + x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16 + case x == 63: + s += 5 + if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. + return decodeErrCodeCorrupt + } + x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24 + } + length = int(x) + 1 + if length <= 0 { + return decodeErrCodeUnsupportedLiteralLength + } + if length > len(dst)-d || length > len(src)-s { + return decodeErrCodeCorrupt + } + copy(dst[d:], src[s:s+length]) + d += length + s += length + continue + + case tagCopy1: + s += 2 + if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. + return decodeErrCodeCorrupt + } + length = 4 + int(src[s-2])>>2&0x7 + offset = int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1])) + + case tagCopy2: + s += 3 + if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. + return decodeErrCodeCorrupt + } + length = 1 + int(src[s-3])>>2 + offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8) + + case tagCopy4: + s += 5 + if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. + return decodeErrCodeCorrupt + } + length = 1 + int(src[s-5])>>2 + offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24) + } + + if offset <= 0 || d < offset || length > len(dst)-d { + return decodeErrCodeCorrupt + } + // Copy from an earlier sub-slice of dst to a later sub-slice. + // If no overlap, use the built-in copy: + if offset > length { + copy(dst[d:d+length], dst[d-offset:]) + d += length + continue + } + + // Unlike the built-in copy function, this byte-by-byte copy always runs + // forwards, even if the slices overlap. Conceptually, this is: + // + // d += forwardCopy(dst[d:d+length], dst[d-offset:]) + // + // We align the slices into a and b and show the compiler they are the same size. + // This allows the loop to run without bounds checks. + a := dst[d : d+length] + b := dst[d-offset:] + b = b[:len(a)] + for i := range a { + a[i] = b[i] + } + d += length + } + if d != len(dst) { + return decodeErrCodeCorrupt + } + return 0 +} diff --git a/vendor/github.com/klauspost/compress/snappy/encode.go b/vendor/github.com/klauspost/compress/snappy/encode.go new file mode 100644 index 000000000..8d393e904 --- /dev/null +++ b/vendor/github.com/klauspost/compress/snappy/encode.go @@ -0,0 +1,285 @@ +// Copyright 2011 The Snappy-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package snappy + +import ( + "encoding/binary" + "errors" + "io" +) + +// Encode returns the encoded form of src. The returned slice may be a sub- +// slice of dst if dst was large enough to hold the entire encoded block. +// Otherwise, a newly allocated slice will be returned. +// +// The dst and src must not overlap. It is valid to pass a nil dst. +func Encode(dst, src []byte) []byte { + if n := MaxEncodedLen(len(src)); n < 0 { + panic(ErrTooLarge) + } else if len(dst) < n { + dst = make([]byte, n) + } + + // The block starts with the varint-encoded length of the decompressed bytes. + d := binary.PutUvarint(dst, uint64(len(src))) + + for len(src) > 0 { + p := src + src = nil + if len(p) > maxBlockSize { + p, src = p[:maxBlockSize], p[maxBlockSize:] + } + if len(p) < minNonLiteralBlockSize { + d += emitLiteral(dst[d:], p) + } else { + d += encodeBlock(dst[d:], p) + } + } + return dst[:d] +} + +// inputMargin is the minimum number of extra input bytes to keep, inside +// encodeBlock's inner loop. On some architectures, this margin lets us +// implement a fast path for emitLiteral, where the copy of short (<= 16 byte) +// literals can be implemented as a single load to and store from a 16-byte +// register. That literal's actual length can be as short as 1 byte, so this +// can copy up to 15 bytes too much, but that's OK as subsequent iterations of +// the encoding loop will fix up the copy overrun, and this inputMargin ensures +// that we don't overrun the dst and src buffers. +const inputMargin = 16 - 1 + +// minNonLiteralBlockSize is the minimum size of the input to encodeBlock that +// could be encoded with a copy tag. This is the minimum with respect to the +// algorithm used by encodeBlock, not a minimum enforced by the file format. +// +// The encoded output must start with at least a 1 byte literal, as there are +// no previous bytes to copy. A minimal (1 byte) copy after that, generated +// from an emitCopy call in encodeBlock's main loop, would require at least +// another inputMargin bytes, for the reason above: we want any emitLiteral +// calls inside encodeBlock's main loop to use the fast path if possible, which +// requires being able to overrun by inputMargin bytes. Thus, +// minNonLiteralBlockSize equals 1 + 1 + inputMargin. +// +// The C++ code doesn't use this exact threshold, but it could, as discussed at +// https://groups.google.com/d/topic/snappy-compression/oGbhsdIJSJ8/discussion +// The difference between Go (2+inputMargin) and C++ (inputMargin) is purely an +// optimization. It should not affect the encoded form. This is tested by +// TestSameEncodingAsCppShortCopies. +const minNonLiteralBlockSize = 1 + 1 + inputMargin + +// MaxEncodedLen returns the maximum length of a snappy block, given its +// uncompressed length. +// +// It will return a negative value if srcLen is too large to encode. +func MaxEncodedLen(srcLen int) int { + n := uint64(srcLen) + if n > 0xffffffff { + return -1 + } + // Compressed data can be defined as: + // compressed := item* literal* + // item := literal* copy + // + // The trailing literal sequence has a space blowup of at most 62/60 + // since a literal of length 60 needs one tag byte + one extra byte + // for length information. + // + // Item blowup is trickier to measure. Suppose the "copy" op copies + // 4 bytes of data. Because of a special check in the encoding code, + // we produce a 4-byte copy only if the offset is < 65536. Therefore + // the copy op takes 3 bytes to encode, and this type of item leads + // to at most the 62/60 blowup for representing literals. + // + // Suppose the "copy" op copies 5 bytes of data. If the offset is big + // enough, it will take 5 bytes to encode the copy op. Therefore the + // worst case here is a one-byte literal followed by a five-byte copy. + // That is, 6 bytes of input turn into 7 bytes of "compressed" data. + // + // This last factor dominates the blowup, so the final estimate is: + n = 32 + n + n/6 + if n > 0xffffffff { + return -1 + } + return int(n) +} + +var errClosed = errors.New("snappy: Writer is closed") + +// NewWriter returns a new Writer that compresses to w. +// +// The Writer returned does not buffer writes. There is no need to Flush or +// Close such a Writer. +// +// Deprecated: the Writer returned is not suitable for many small writes, only +// for few large writes. Use NewBufferedWriter instead, which is efficient +// regardless of the frequency and shape of the writes, and remember to Close +// that Writer when done. +func NewWriter(w io.Writer) *Writer { + return &Writer{ + w: w, + obuf: make([]byte, obufLen), + } +} + +// NewBufferedWriter returns a new Writer that compresses to w, using the +// framing format described at +// https://github.com/google/snappy/blob/master/framing_format.txt +// +// The Writer returned buffers writes. Users must call Close to guarantee all +// data has been forwarded to the underlying io.Writer. They may also call +// Flush zero or more times before calling Close. +func NewBufferedWriter(w io.Writer) *Writer { + return &Writer{ + w: w, + ibuf: make([]byte, 0, maxBlockSize), + obuf: make([]byte, obufLen), + } +} + +// Writer is an io.Writer that can write Snappy-compressed bytes. +type Writer struct { + w io.Writer + err error + + // ibuf is a buffer for the incoming (uncompressed) bytes. + // + // Its use is optional. For backwards compatibility, Writers created by the + // NewWriter function have ibuf == nil, do not buffer incoming bytes, and + // therefore do not need to be Flush'ed or Close'd. + ibuf []byte + + // obuf is a buffer for the outgoing (compressed) bytes. + obuf []byte + + // wroteStreamHeader is whether we have written the stream header. + wroteStreamHeader bool +} + +// Reset discards the writer's state and switches the Snappy writer to write to +// w. This permits reusing a Writer rather than allocating a new one. +func (w *Writer) Reset(writer io.Writer) { + w.w = writer + w.err = nil + if w.ibuf != nil { + w.ibuf = w.ibuf[:0] + } + w.wroteStreamHeader = false +} + +// Write satisfies the io.Writer interface. +func (w *Writer) Write(p []byte) (nRet int, errRet error) { + if w.ibuf == nil { + // Do not buffer incoming bytes. This does not perform or compress well + // if the caller of Writer.Write writes many small slices. This + // behavior is therefore deprecated, but still supported for backwards + // compatibility with code that doesn't explicitly Flush or Close. + return w.write(p) + } + + // The remainder of this method is based on bufio.Writer.Write from the + // standard library. + + for len(p) > (cap(w.ibuf)-len(w.ibuf)) && w.err == nil { + var n int + if len(w.ibuf) == 0 { + // Large write, empty buffer. + // Write directly from p to avoid copy. + n, _ = w.write(p) + } else { + n = copy(w.ibuf[len(w.ibuf):cap(w.ibuf)], p) + w.ibuf = w.ibuf[:len(w.ibuf)+n] + w.Flush() + } + nRet += n + p = p[n:] + } + if w.err != nil { + return nRet, w.err + } + n := copy(w.ibuf[len(w.ibuf):cap(w.ibuf)], p) + w.ibuf = w.ibuf[:len(w.ibuf)+n] + nRet += n + return nRet, nil +} + +func (w *Writer) write(p []byte) (nRet int, errRet error) { + if w.err != nil { + return 0, w.err + } + for len(p) > 0 { + obufStart := len(magicChunk) + if !w.wroteStreamHeader { + w.wroteStreamHeader = true + copy(w.obuf, magicChunk) + obufStart = 0 + } + + var uncompressed []byte + if len(p) > maxBlockSize { + uncompressed, p = p[:maxBlockSize], p[maxBlockSize:] + } else { + uncompressed, p = p, nil + } + checksum := crc(uncompressed) + + // Compress the buffer, discarding the result if the improvement + // isn't at least 12.5%. + compressed := Encode(w.obuf[obufHeaderLen:], uncompressed) + chunkType := uint8(chunkTypeCompressedData) + chunkLen := 4 + len(compressed) + obufEnd := obufHeaderLen + len(compressed) + if len(compressed) >= len(uncompressed)-len(uncompressed)/8 { + chunkType = chunkTypeUncompressedData + chunkLen = 4 + len(uncompressed) + obufEnd = obufHeaderLen + } + + // Fill in the per-chunk header that comes before the body. + w.obuf[len(magicChunk)+0] = chunkType + w.obuf[len(magicChunk)+1] = uint8(chunkLen >> 0) + w.obuf[len(magicChunk)+2] = uint8(chunkLen >> 8) + w.obuf[len(magicChunk)+3] = uint8(chunkLen >> 16) + w.obuf[len(magicChunk)+4] = uint8(checksum >> 0) + w.obuf[len(magicChunk)+5] = uint8(checksum >> 8) + w.obuf[len(magicChunk)+6] = uint8(checksum >> 16) + w.obuf[len(magicChunk)+7] = uint8(checksum >> 24) + + if _, err := w.w.Write(w.obuf[obufStart:obufEnd]); err != nil { + w.err = err + return nRet, err + } + if chunkType == chunkTypeUncompressedData { + if _, err := w.w.Write(uncompressed); err != nil { + w.err = err + return nRet, err + } + } + nRet += len(uncompressed) + } + return nRet, nil +} + +// Flush flushes the Writer to its underlying io.Writer. +func (w *Writer) Flush() error { + if w.err != nil { + return w.err + } + if len(w.ibuf) == 0 { + return nil + } + w.write(w.ibuf) + w.ibuf = w.ibuf[:0] + return w.err +} + +// Close calls Flush and then closes the Writer. +func (w *Writer) Close() error { + w.Flush() + ret := w.err + if w.err == nil { + w.err = errClosed + } + return ret +} diff --git a/vendor/github.com/klauspost/compress/snappy/encode_amd64.go b/vendor/github.com/klauspost/compress/snappy/encode_amd64.go new file mode 100644 index 000000000..150d91bc8 --- /dev/null +++ b/vendor/github.com/klauspost/compress/snappy/encode_amd64.go @@ -0,0 +1,29 @@ +// Copyright 2016 The Snappy-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !appengine +// +build gc +// +build !noasm + +package snappy + +// emitLiteral has the same semantics as in encode_other.go. +// +//go:noescape +func emitLiteral(dst, lit []byte) int + +// emitCopy has the same semantics as in encode_other.go. +// +//go:noescape +func emitCopy(dst []byte, offset, length int) int + +// extendMatch has the same semantics as in encode_other.go. +// +//go:noescape +func extendMatch(src []byte, i, j int) int + +// encodeBlock has the same semantics as in encode_other.go. +// +//go:noescape +func encodeBlock(dst, src []byte) (d int) diff --git a/vendor/github.com/klauspost/compress/snappy/encode_amd64.s b/vendor/github.com/klauspost/compress/snappy/encode_amd64.s new file mode 100644 index 000000000..adfd979fe --- /dev/null +++ b/vendor/github.com/klauspost/compress/snappy/encode_amd64.s @@ -0,0 +1,730 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !appengine +// +build gc +// +build !noasm + +#include "textflag.h" + +// The XXX lines assemble on Go 1.4, 1.5 and 1.7, but not 1.6, due to a +// Go toolchain regression. See https://github.com/golang/go/issues/15426 and +// https://github.com/golang/snappy/issues/29 +// +// As a workaround, the package was built with a known good assembler, and +// those instructions were disassembled by "objdump -d" to yield the +// 4e 0f b7 7c 5c 78 movzwq 0x78(%rsp,%r11,2),%r15 +// style comments, in AT&T asm syntax. Note that rsp here is a physical +// register, not Go/asm's SP pseudo-register (see https://golang.org/doc/asm). +// The instructions were then encoded as "BYTE $0x.." sequences, which assemble +// fine on Go 1.6. + +// The asm code generally follows the pure Go code in encode_other.go, except +// where marked with a "!!!". + +// ---------------------------------------------------------------------------- + +// func emitLiteral(dst, lit []byte) int +// +// All local variables fit into registers. The register allocation: +// - AX len(lit) +// - BX n +// - DX return value +// - DI &dst[i] +// - R10 &lit[0] +// +// The 24 bytes of stack space is to call runtime·memmove. +// +// The unusual register allocation of local variables, such as R10 for the +// source pointer, matches the allocation used at the call site in encodeBlock, +// which makes it easier to manually inline this function. +TEXT ·emitLiteral(SB), NOSPLIT, $24-56 + MOVQ dst_base+0(FP), DI + MOVQ lit_base+24(FP), R10 + MOVQ lit_len+32(FP), AX + MOVQ AX, DX + MOVL AX, BX + SUBL $1, BX + + CMPL BX, $60 + JLT oneByte + CMPL BX, $256 + JLT twoBytes + +threeBytes: + MOVB $0xf4, 0(DI) + MOVW BX, 1(DI) + ADDQ $3, DI + ADDQ $3, DX + JMP memmove + +twoBytes: + MOVB $0xf0, 0(DI) + MOVB BX, 1(DI) + ADDQ $2, DI + ADDQ $2, DX + JMP memmove + +oneByte: + SHLB $2, BX + MOVB BX, 0(DI) + ADDQ $1, DI + ADDQ $1, DX + +memmove: + MOVQ DX, ret+48(FP) + + // copy(dst[i:], lit) + // + // This means calling runtime·memmove(&dst[i], &lit[0], len(lit)), so we push + // DI, R10 and AX as arguments. + MOVQ DI, 0(SP) + MOVQ R10, 8(SP) + MOVQ AX, 16(SP) + CALL runtime·memmove(SB) + RET + +// ---------------------------------------------------------------------------- + +// func emitCopy(dst []byte, offset, length int) int +// +// All local variables fit into registers. The register allocation: +// - AX length +// - SI &dst[0] +// - DI &dst[i] +// - R11 offset +// +// The unusual register allocation of local variables, such as R11 for the +// offset, matches the allocation used at the call site in encodeBlock, which +// makes it easier to manually inline this function. +TEXT ·emitCopy(SB), NOSPLIT, $0-48 + MOVQ dst_base+0(FP), DI + MOVQ DI, SI + MOVQ offset+24(FP), R11 + MOVQ length+32(FP), AX + +loop0: + // for length >= 68 { etc } + CMPL AX, $68 + JLT step1 + + // Emit a length 64 copy, encoded as 3 bytes. + MOVB $0xfe, 0(DI) + MOVW R11, 1(DI) + ADDQ $3, DI + SUBL $64, AX + JMP loop0 + +step1: + // if length > 64 { etc } + CMPL AX, $64 + JLE step2 + + // Emit a length 60 copy, encoded as 3 bytes. + MOVB $0xee, 0(DI) + MOVW R11, 1(DI) + ADDQ $3, DI + SUBL $60, AX + +step2: + // if length >= 12 || offset >= 2048 { goto step3 } + CMPL AX, $12 + JGE step3 + CMPL R11, $2048 + JGE step3 + + // Emit the remaining copy, encoded as 2 bytes. + MOVB R11, 1(DI) + SHRL $8, R11 + SHLB $5, R11 + SUBB $4, AX + SHLB $2, AX + ORB AX, R11 + ORB $1, R11 + MOVB R11, 0(DI) + ADDQ $2, DI + + // Return the number of bytes written. + SUBQ SI, DI + MOVQ DI, ret+40(FP) + RET + +step3: + // Emit the remaining copy, encoded as 3 bytes. + SUBL $1, AX + SHLB $2, AX + ORB $2, AX + MOVB AX, 0(DI) + MOVW R11, 1(DI) + ADDQ $3, DI + + // Return the number of bytes written. + SUBQ SI, DI + MOVQ DI, ret+40(FP) + RET + +// ---------------------------------------------------------------------------- + +// func extendMatch(src []byte, i, j int) int +// +// All local variables fit into registers. The register allocation: +// - DX &src[0] +// - SI &src[j] +// - R13 &src[len(src) - 8] +// - R14 &src[len(src)] +// - R15 &src[i] +// +// The unusual register allocation of local variables, such as R15 for a source +// pointer, matches the allocation used at the call site in encodeBlock, which +// makes it easier to manually inline this function. +TEXT ·extendMatch(SB), NOSPLIT, $0-48 + MOVQ src_base+0(FP), DX + MOVQ src_len+8(FP), R14 + MOVQ i+24(FP), R15 + MOVQ j+32(FP), SI + ADDQ DX, R14 + ADDQ DX, R15 + ADDQ DX, SI + MOVQ R14, R13 + SUBQ $8, R13 + +cmp8: + // As long as we are 8 or more bytes before the end of src, we can load and + // compare 8 bytes at a time. If those 8 bytes are equal, repeat. + CMPQ SI, R13 + JA cmp1 + MOVQ (R15), AX + MOVQ (SI), BX + CMPQ AX, BX + JNE bsf + ADDQ $8, R15 + ADDQ $8, SI + JMP cmp8 + +bsf: + // If those 8 bytes were not equal, XOR the two 8 byte values, and return + // the index of the first byte that differs. The BSF instruction finds the + // least significant 1 bit, the amd64 architecture is little-endian, and + // the shift by 3 converts a bit index to a byte index. + XORQ AX, BX + BSFQ BX, BX + SHRQ $3, BX + ADDQ BX, SI + + // Convert from &src[ret] to ret. + SUBQ DX, SI + MOVQ SI, ret+40(FP) + RET + +cmp1: + // In src's tail, compare 1 byte at a time. + CMPQ SI, R14 + JAE extendMatchEnd + MOVB (R15), AX + MOVB (SI), BX + CMPB AX, BX + JNE extendMatchEnd + ADDQ $1, R15 + ADDQ $1, SI + JMP cmp1 + +extendMatchEnd: + // Convert from &src[ret] to ret. + SUBQ DX, SI + MOVQ SI, ret+40(FP) + RET + +// ---------------------------------------------------------------------------- + +// func encodeBlock(dst, src []byte) (d int) +// +// All local variables fit into registers, other than "var table". The register +// allocation: +// - AX . . +// - BX . . +// - CX 56 shift (note that amd64 shifts by non-immediates must use CX). +// - DX 64 &src[0], tableSize +// - SI 72 &src[s] +// - DI 80 &dst[d] +// - R9 88 sLimit +// - R10 . &src[nextEmit] +// - R11 96 prevHash, currHash, nextHash, offset +// - R12 104 &src[base], skip +// - R13 . &src[nextS], &src[len(src) - 8] +// - R14 . len(src), bytesBetweenHashLookups, &src[len(src)], x +// - R15 112 candidate +// +// The second column (56, 64, etc) is the stack offset to spill the registers +// when calling other functions. We could pack this slightly tighter, but it's +// simpler to have a dedicated spill map independent of the function called. +// +// "var table [maxTableSize]uint16" takes up 32768 bytes of stack space. An +// extra 56 bytes, to call other functions, and an extra 64 bytes, to spill +// local variables (registers) during calls gives 32768 + 56 + 64 = 32888. +TEXT ·encodeBlock(SB), 0, $32888-56 + MOVQ dst_base+0(FP), DI + MOVQ src_base+24(FP), SI + MOVQ src_len+32(FP), R14 + + // shift, tableSize := uint32(32-8), 1<<8 + MOVQ $24, CX + MOVQ $256, DX + +calcShift: + // for ; tableSize < maxTableSize && tableSize < len(src); tableSize *= 2 { + // shift-- + // } + CMPQ DX, $16384 + JGE varTable + CMPQ DX, R14 + JGE varTable + SUBQ $1, CX + SHLQ $1, DX + JMP calcShift + +varTable: + // var table [maxTableSize]uint16 + // + // In the asm code, unlike the Go code, we can zero-initialize only the + // first tableSize elements. Each uint16 element is 2 bytes and each MOVOU + // writes 16 bytes, so we can do only tableSize/8 writes instead of the + // 2048 writes that would zero-initialize all of table's 32768 bytes. + SHRQ $3, DX + LEAQ table-32768(SP), BX + PXOR X0, X0 + +memclr: + MOVOU X0, 0(BX) + ADDQ $16, BX + SUBQ $1, DX + JNZ memclr + + // !!! DX = &src[0] + MOVQ SI, DX + + // sLimit := len(src) - inputMargin + MOVQ R14, R9 + SUBQ $15, R9 + + // !!! Pre-emptively spill CX, DX and R9 to the stack. Their values don't + // change for the rest of the function. + MOVQ CX, 56(SP) + MOVQ DX, 64(SP) + MOVQ R9, 88(SP) + + // nextEmit := 0 + MOVQ DX, R10 + + // s := 1 + ADDQ $1, SI + + // nextHash := hash(load32(src, s), shift) + MOVL 0(SI), R11 + IMULL $0x1e35a7bd, R11 + SHRL CX, R11 + +outer: + // for { etc } + + // skip := 32 + MOVQ $32, R12 + + // nextS := s + MOVQ SI, R13 + + // candidate := 0 + MOVQ $0, R15 + +inner0: + // for { etc } + + // s := nextS + MOVQ R13, SI + + // bytesBetweenHashLookups := skip >> 5 + MOVQ R12, R14 + SHRQ $5, R14 + + // nextS = s + bytesBetweenHashLookups + ADDQ R14, R13 + + // skip += bytesBetweenHashLookups + ADDQ R14, R12 + + // if nextS > sLimit { goto emitRemainder } + MOVQ R13, AX + SUBQ DX, AX + CMPQ AX, R9 + JA emitRemainder + + // candidate = int(table[nextHash]) + // XXX: MOVWQZX table-32768(SP)(R11*2), R15 + // XXX: 4e 0f b7 7c 5c 78 movzwq 0x78(%rsp,%r11,2),%r15 + BYTE $0x4e + BYTE $0x0f + BYTE $0xb7 + BYTE $0x7c + BYTE $0x5c + BYTE $0x78 + + // table[nextHash] = uint16(s) + MOVQ SI, AX + SUBQ DX, AX + + // XXX: MOVW AX, table-32768(SP)(R11*2) + // XXX: 66 42 89 44 5c 78 mov %ax,0x78(%rsp,%r11,2) + BYTE $0x66 + BYTE $0x42 + BYTE $0x89 + BYTE $0x44 + BYTE $0x5c + BYTE $0x78 + + // nextHash = hash(load32(src, nextS), shift) + MOVL 0(R13), R11 + IMULL $0x1e35a7bd, R11 + SHRL CX, R11 + + // if load32(src, s) != load32(src, candidate) { continue } break + MOVL 0(SI), AX + MOVL (DX)(R15*1), BX + CMPL AX, BX + JNE inner0 + +fourByteMatch: + // As per the encode_other.go code: + // + // A 4-byte match has been found. We'll later see etc. + + // !!! Jump to a fast path for short (<= 16 byte) literals. See the comment + // on inputMargin in encode.go. + MOVQ SI, AX + SUBQ R10, AX + CMPQ AX, $16 + JLE emitLiteralFastPath + + // ---------------------------------------- + // Begin inline of the emitLiteral call. + // + // d += emitLiteral(dst[d:], src[nextEmit:s]) + + MOVL AX, BX + SUBL $1, BX + + CMPL BX, $60 + JLT inlineEmitLiteralOneByte + CMPL BX, $256 + JLT inlineEmitLiteralTwoBytes + +inlineEmitLiteralThreeBytes: + MOVB $0xf4, 0(DI) + MOVW BX, 1(DI) + ADDQ $3, DI + JMP inlineEmitLiteralMemmove + +inlineEmitLiteralTwoBytes: + MOVB $0xf0, 0(DI) + MOVB BX, 1(DI) + ADDQ $2, DI + JMP inlineEmitLiteralMemmove + +inlineEmitLiteralOneByte: + SHLB $2, BX + MOVB BX, 0(DI) + ADDQ $1, DI + +inlineEmitLiteralMemmove: + // Spill local variables (registers) onto the stack; call; unspill. + // + // copy(dst[i:], lit) + // + // This means calling runtime·memmove(&dst[i], &lit[0], len(lit)), so we push + // DI, R10 and AX as arguments. + MOVQ DI, 0(SP) + MOVQ R10, 8(SP) + MOVQ AX, 16(SP) + ADDQ AX, DI // Finish the "d +=" part of "d += emitLiteral(etc)". + MOVQ SI, 72(SP) + MOVQ DI, 80(SP) + MOVQ R15, 112(SP) + CALL runtime·memmove(SB) + MOVQ 56(SP), CX + MOVQ 64(SP), DX + MOVQ 72(SP), SI + MOVQ 80(SP), DI + MOVQ 88(SP), R9 + MOVQ 112(SP), R15 + JMP inner1 + +inlineEmitLiteralEnd: + // End inline of the emitLiteral call. + // ---------------------------------------- + +emitLiteralFastPath: + // !!! Emit the 1-byte encoding "uint8(len(lit)-1)<<2". + MOVB AX, BX + SUBB $1, BX + SHLB $2, BX + MOVB BX, (DI) + ADDQ $1, DI + + // !!! Implement the copy from lit to dst as a 16-byte load and store. + // (Encode's documentation says that dst and src must not overlap.) + // + // This always copies 16 bytes, instead of only len(lit) bytes, but that's + // OK. Subsequent iterations will fix up the overrun. + // + // Note that on amd64, it is legal and cheap to issue unaligned 8-byte or + // 16-byte loads and stores. This technique probably wouldn't be as + // effective on architectures that are fussier about alignment. + MOVOU 0(R10), X0 + MOVOU X0, 0(DI) + ADDQ AX, DI + +inner1: + // for { etc } + + // base := s + MOVQ SI, R12 + + // !!! offset := base - candidate + MOVQ R12, R11 + SUBQ R15, R11 + SUBQ DX, R11 + + // ---------------------------------------- + // Begin inline of the extendMatch call. + // + // s = extendMatch(src, candidate+4, s+4) + + // !!! R14 = &src[len(src)] + MOVQ src_len+32(FP), R14 + ADDQ DX, R14 + + // !!! R13 = &src[len(src) - 8] + MOVQ R14, R13 + SUBQ $8, R13 + + // !!! R15 = &src[candidate + 4] + ADDQ $4, R15 + ADDQ DX, R15 + + // !!! s += 4 + ADDQ $4, SI + +inlineExtendMatchCmp8: + // As long as we are 8 or more bytes before the end of src, we can load and + // compare 8 bytes at a time. If those 8 bytes are equal, repeat. + CMPQ SI, R13 + JA inlineExtendMatchCmp1 + MOVQ (R15), AX + MOVQ (SI), BX + CMPQ AX, BX + JNE inlineExtendMatchBSF + ADDQ $8, R15 + ADDQ $8, SI + JMP inlineExtendMatchCmp8 + +inlineExtendMatchBSF: + // If those 8 bytes were not equal, XOR the two 8 byte values, and return + // the index of the first byte that differs. The BSF instruction finds the + // least significant 1 bit, the amd64 architecture is little-endian, and + // the shift by 3 converts a bit index to a byte index. + XORQ AX, BX + BSFQ BX, BX + SHRQ $3, BX + ADDQ BX, SI + JMP inlineExtendMatchEnd + +inlineExtendMatchCmp1: + // In src's tail, compare 1 byte at a time. + CMPQ SI, R14 + JAE inlineExtendMatchEnd + MOVB (R15), AX + MOVB (SI), BX + CMPB AX, BX + JNE inlineExtendMatchEnd + ADDQ $1, R15 + ADDQ $1, SI + JMP inlineExtendMatchCmp1 + +inlineExtendMatchEnd: + // End inline of the extendMatch call. + // ---------------------------------------- + + // ---------------------------------------- + // Begin inline of the emitCopy call. + // + // d += emitCopy(dst[d:], base-candidate, s-base) + + // !!! length := s - base + MOVQ SI, AX + SUBQ R12, AX + +inlineEmitCopyLoop0: + // for length >= 68 { etc } + CMPL AX, $68 + JLT inlineEmitCopyStep1 + + // Emit a length 64 copy, encoded as 3 bytes. + MOVB $0xfe, 0(DI) + MOVW R11, 1(DI) + ADDQ $3, DI + SUBL $64, AX + JMP inlineEmitCopyLoop0 + +inlineEmitCopyStep1: + // if length > 64 { etc } + CMPL AX, $64 + JLE inlineEmitCopyStep2 + + // Emit a length 60 copy, encoded as 3 bytes. + MOVB $0xee, 0(DI) + MOVW R11, 1(DI) + ADDQ $3, DI + SUBL $60, AX + +inlineEmitCopyStep2: + // if length >= 12 || offset >= 2048 { goto inlineEmitCopyStep3 } + CMPL AX, $12 + JGE inlineEmitCopyStep3 + CMPL R11, $2048 + JGE inlineEmitCopyStep3 + + // Emit the remaining copy, encoded as 2 bytes. + MOVB R11, 1(DI) + SHRL $8, R11 + SHLB $5, R11 + SUBB $4, AX + SHLB $2, AX + ORB AX, R11 + ORB $1, R11 + MOVB R11, 0(DI) + ADDQ $2, DI + JMP inlineEmitCopyEnd + +inlineEmitCopyStep3: + // Emit the remaining copy, encoded as 3 bytes. + SUBL $1, AX + SHLB $2, AX + ORB $2, AX + MOVB AX, 0(DI) + MOVW R11, 1(DI) + ADDQ $3, DI + +inlineEmitCopyEnd: + // End inline of the emitCopy call. + // ---------------------------------------- + + // nextEmit = s + MOVQ SI, R10 + + // if s >= sLimit { goto emitRemainder } + MOVQ SI, AX + SUBQ DX, AX + CMPQ AX, R9 + JAE emitRemainder + + // As per the encode_other.go code: + // + // We could immediately etc. + + // x := load64(src, s-1) + MOVQ -1(SI), R14 + + // prevHash := hash(uint32(x>>0), shift) + MOVL R14, R11 + IMULL $0x1e35a7bd, R11 + SHRL CX, R11 + + // table[prevHash] = uint16(s-1) + MOVQ SI, AX + SUBQ DX, AX + SUBQ $1, AX + + // XXX: MOVW AX, table-32768(SP)(R11*2) + // XXX: 66 42 89 44 5c 78 mov %ax,0x78(%rsp,%r11,2) + BYTE $0x66 + BYTE $0x42 + BYTE $0x89 + BYTE $0x44 + BYTE $0x5c + BYTE $0x78 + + // currHash := hash(uint32(x>>8), shift) + SHRQ $8, R14 + MOVL R14, R11 + IMULL $0x1e35a7bd, R11 + SHRL CX, R11 + + // candidate = int(table[currHash]) + // XXX: MOVWQZX table-32768(SP)(R11*2), R15 + // XXX: 4e 0f b7 7c 5c 78 movzwq 0x78(%rsp,%r11,2),%r15 + BYTE $0x4e + BYTE $0x0f + BYTE $0xb7 + BYTE $0x7c + BYTE $0x5c + BYTE $0x78 + + // table[currHash] = uint16(s) + ADDQ $1, AX + + // XXX: MOVW AX, table-32768(SP)(R11*2) + // XXX: 66 42 89 44 5c 78 mov %ax,0x78(%rsp,%r11,2) + BYTE $0x66 + BYTE $0x42 + BYTE $0x89 + BYTE $0x44 + BYTE $0x5c + BYTE $0x78 + + // if uint32(x>>8) == load32(src, candidate) { continue } + MOVL (DX)(R15*1), BX + CMPL R14, BX + JEQ inner1 + + // nextHash = hash(uint32(x>>16), shift) + SHRQ $8, R14 + MOVL R14, R11 + IMULL $0x1e35a7bd, R11 + SHRL CX, R11 + + // s++ + ADDQ $1, SI + + // break out of the inner1 for loop, i.e. continue the outer loop. + JMP outer + +emitRemainder: + // if nextEmit < len(src) { etc } + MOVQ src_len+32(FP), AX + ADDQ DX, AX + CMPQ R10, AX + JEQ encodeBlockEnd + + // d += emitLiteral(dst[d:], src[nextEmit:]) + // + // Push args. + MOVQ DI, 0(SP) + MOVQ $0, 8(SP) // Unnecessary, as the callee ignores it, but conservative. + MOVQ $0, 16(SP) // Unnecessary, as the callee ignores it, but conservative. + MOVQ R10, 24(SP) + SUBQ R10, AX + MOVQ AX, 32(SP) + MOVQ AX, 40(SP) // Unnecessary, as the callee ignores it, but conservative. + + // Spill local variables (registers) onto the stack; call; unspill. + MOVQ DI, 80(SP) + CALL ·emitLiteral(SB) + MOVQ 80(SP), DI + + // Finish the "d +=" part of "d += emitLiteral(etc)". + ADDQ 48(SP), DI + +encodeBlockEnd: + MOVQ dst_base+0(FP), AX + SUBQ AX, DI + MOVQ DI, d+48(FP) + RET diff --git a/vendor/github.com/klauspost/compress/snappy/encode_other.go b/vendor/github.com/klauspost/compress/snappy/encode_other.go new file mode 100644 index 000000000..dbcae905e --- /dev/null +++ b/vendor/github.com/klauspost/compress/snappy/encode_other.go @@ -0,0 +1,238 @@ +// Copyright 2016 The Snappy-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !amd64 appengine !gc noasm + +package snappy + +func load32(b []byte, i int) uint32 { + b = b[i : i+4 : len(b)] // Help the compiler eliminate bounds checks on the next line. + return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 +} + +func load64(b []byte, i int) uint64 { + b = b[i : i+8 : len(b)] // Help the compiler eliminate bounds checks on the next line. + return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | + uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 +} + +// emitLiteral writes a literal chunk and returns the number of bytes written. +// +// It assumes that: +// dst is long enough to hold the encoded bytes +// 1 <= len(lit) && len(lit) <= 65536 +func emitLiteral(dst, lit []byte) int { + i, n := 0, uint(len(lit)-1) + switch { + case n < 60: + dst[0] = uint8(n)<<2 | tagLiteral + i = 1 + case n < 1<<8: + dst[0] = 60<<2 | tagLiteral + dst[1] = uint8(n) + i = 2 + default: + dst[0] = 61<<2 | tagLiteral + dst[1] = uint8(n) + dst[2] = uint8(n >> 8) + i = 3 + } + return i + copy(dst[i:], lit) +} + +// emitCopy writes a copy chunk and returns the number of bytes written. +// +// It assumes that: +// dst is long enough to hold the encoded bytes +// 1 <= offset && offset <= 65535 +// 4 <= length && length <= 65535 +func emitCopy(dst []byte, offset, length int) int { + i := 0 + // The maximum length for a single tagCopy1 or tagCopy2 op is 64 bytes. The + // threshold for this loop is a little higher (at 68 = 64 + 4), and the + // length emitted down below is is a little lower (at 60 = 64 - 4), because + // it's shorter to encode a length 67 copy as a length 60 tagCopy2 followed + // by a length 7 tagCopy1 (which encodes as 3+2 bytes) than to encode it as + // a length 64 tagCopy2 followed by a length 3 tagCopy2 (which encodes as + // 3+3 bytes). The magic 4 in the 64±4 is because the minimum length for a + // tagCopy1 op is 4 bytes, which is why a length 3 copy has to be an + // encodes-as-3-bytes tagCopy2 instead of an encodes-as-2-bytes tagCopy1. + for length >= 68 { + // Emit a length 64 copy, encoded as 3 bytes. + dst[i+0] = 63<<2 | tagCopy2 + dst[i+1] = uint8(offset) + dst[i+2] = uint8(offset >> 8) + i += 3 + length -= 64 + } + if length > 64 { + // Emit a length 60 copy, encoded as 3 bytes. + dst[i+0] = 59<<2 | tagCopy2 + dst[i+1] = uint8(offset) + dst[i+2] = uint8(offset >> 8) + i += 3 + length -= 60 + } + if length >= 12 || offset >= 2048 { + // Emit the remaining copy, encoded as 3 bytes. + dst[i+0] = uint8(length-1)<<2 | tagCopy2 + dst[i+1] = uint8(offset) + dst[i+2] = uint8(offset >> 8) + return i + 3 + } + // Emit the remaining copy, encoded as 2 bytes. + dst[i+0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1 + dst[i+1] = uint8(offset) + return i + 2 +} + +// extendMatch returns the largest k such that k <= len(src) and that +// src[i:i+k-j] and src[j:k] have the same contents. +// +// It assumes that: +// 0 <= i && i < j && j <= len(src) +func extendMatch(src []byte, i, j int) int { + for ; j < len(src) && src[i] == src[j]; i, j = i+1, j+1 { + } + return j +} + +func hash(u, shift uint32) uint32 { + return (u * 0x1e35a7bd) >> shift +} + +// encodeBlock encodes a non-empty src to a guaranteed-large-enough dst. It +// assumes that the varint-encoded length of the decompressed bytes has already +// been written. +// +// It also assumes that: +// len(dst) >= MaxEncodedLen(len(src)) && +// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize +func encodeBlock(dst, src []byte) (d int) { + // Initialize the hash table. Its size ranges from 1<<8 to 1<<14 inclusive. + // The table element type is uint16, as s < sLimit and sLimit < len(src) + // and len(src) <= maxBlockSize and maxBlockSize == 65536. + const ( + maxTableSize = 1 << 14 + // tableMask is redundant, but helps the compiler eliminate bounds + // checks. + tableMask = maxTableSize - 1 + ) + shift := uint32(32 - 8) + for tableSize := 1 << 8; tableSize < maxTableSize && tableSize < len(src); tableSize *= 2 { + shift-- + } + // In Go, all array elements are zero-initialized, so there is no advantage + // to a smaller tableSize per se. However, it matches the C++ algorithm, + // and in the asm versions of this code, we can get away with zeroing only + // the first tableSize elements. + var table [maxTableSize]uint16 + + // sLimit is when to stop looking for offset/length copies. The inputMargin + // lets us use a fast path for emitLiteral in the main loop, while we are + // looking for copies. + sLimit := len(src) - inputMargin + + // nextEmit is where in src the next emitLiteral should start from. + nextEmit := 0 + + // The encoded form must start with a literal, as there are no previous + // bytes to copy, so we start looking for hash matches at s == 1. + s := 1 + nextHash := hash(load32(src, s), shift) + + for { + // Copied from the C++ snappy implementation: + // + // Heuristic match skipping: If 32 bytes are scanned with no matches + // found, start looking only at every other byte. If 32 more bytes are + // scanned (or skipped), look at every third byte, etc.. When a match + // is found, immediately go back to looking at every byte. This is a + // small loss (~5% performance, ~0.1% density) for compressible data + // due to more bookkeeping, but for non-compressible data (such as + // JPEG) it's a huge win since the compressor quickly "realizes" the + // data is incompressible and doesn't bother looking for matches + // everywhere. + // + // The "skip" variable keeps track of how many bytes there are since + // the last match; dividing it by 32 (ie. right-shifting by five) gives + // the number of bytes to move ahead for each iteration. + skip := 32 + + nextS := s + candidate := 0 + for { + s = nextS + bytesBetweenHashLookups := skip >> 5 + nextS = s + bytesBetweenHashLookups + skip += bytesBetweenHashLookups + if nextS > sLimit { + goto emitRemainder + } + candidate = int(table[nextHash&tableMask]) + table[nextHash&tableMask] = uint16(s) + nextHash = hash(load32(src, nextS), shift) + if load32(src, s) == load32(src, candidate) { + break + } + } + + // A 4-byte match has been found. We'll later see if more than 4 bytes + // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit + // them as literal bytes. + d += emitLiteral(dst[d:], src[nextEmit:s]) + + // Call emitCopy, and then see if another emitCopy could be our next + // move. Repeat until we find no match for the input immediately after + // what was consumed by the last emitCopy call. + // + // If we exit this loop normally then we need to call emitLiteral next, + // though we don't yet know how big the literal will be. We handle that + // by proceeding to the next iteration of the main loop. We also can + // exit this loop via goto if we get close to exhausting the input. + for { + // Invariant: we have a 4-byte match at s, and no need to emit any + // literal bytes prior to s. + base := s + + // Extend the 4-byte match as long as possible. + // + // This is an inlined version of: + // s = extendMatch(src, candidate+4, s+4) + s += 4 + for i := candidate + 4; s < len(src) && src[i] == src[s]; i, s = i+1, s+1 { + } + + d += emitCopy(dst[d:], base-candidate, s-base) + nextEmit = s + if s >= sLimit { + goto emitRemainder + } + + // We could immediately start working at s now, but to improve + // compression we first update the hash table at s-1 and at s. If + // another emitCopy is not our next move, also calculate nextHash + // at s+1. At least on GOARCH=amd64, these three hash calculations + // are faster as one load64 call (with some shifts) instead of + // three load32 calls. + x := load64(src, s-1) + prevHash := hash(uint32(x>>0), shift) + table[prevHash&tableMask] = uint16(s - 1) + currHash := hash(uint32(x>>8), shift) + candidate = int(table[currHash&tableMask]) + table[currHash&tableMask] = uint16(s) + if uint32(x>>8) != load32(src, candidate) { + nextHash = hash(uint32(x>>16), shift) + s++ + break + } + } + } + +emitRemainder: + if nextEmit < len(src) { + d += emitLiteral(dst[d:], src[nextEmit:]) + } + return d +} diff --git a/vendor/github.com/klauspost/compress/snappy/runbench.cmd b/vendor/github.com/klauspost/compress/snappy/runbench.cmd new file mode 100644 index 000000000..d24eb4b47 --- /dev/null +++ b/vendor/github.com/klauspost/compress/snappy/runbench.cmd @@ -0,0 +1,2 @@ +del old.txt +go test -bench=. >>old.txt && go test -bench=. >>old.txt && go test -bench=. >>old.txt && benchstat -delta-test=ttest old.txt new.txt diff --git a/vendor/github.com/klauspost/compress/snappy/snappy.go b/vendor/github.com/klauspost/compress/snappy/snappy.go new file mode 100644 index 000000000..74a36689e --- /dev/null +++ b/vendor/github.com/klauspost/compress/snappy/snappy.go @@ -0,0 +1,98 @@ +// Copyright 2011 The Snappy-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package snappy implements the Snappy compression format. It aims for very +// high speeds and reasonable compression. +// +// There are actually two Snappy formats: block and stream. They are related, +// but different: trying to decompress block-compressed data as a Snappy stream +// will fail, and vice versa. The block format is the Decode and Encode +// functions and the stream format is the Reader and Writer types. +// +// The block format, the more common case, is used when the complete size (the +// number of bytes) of the original data is known upfront, at the time +// compression starts. The stream format, also known as the framing format, is +// for when that isn't always true. +// +// The canonical, C++ implementation is at https://github.com/google/snappy and +// it only implements the block format. +package snappy + +import ( + "hash/crc32" +) + +/* +Each encoded block begins with the varint-encoded length of the decoded data, +followed by a sequence of chunks. Chunks begin and end on byte boundaries. The +first byte of each chunk is broken into its 2 least and 6 most significant bits +called l and m: l ranges in [0, 4) and m ranges in [0, 64). l is the chunk tag. +Zero means a literal tag. All other values mean a copy tag. + +For literal tags: + - If m < 60, the next 1 + m bytes are literal bytes. + - Otherwise, let n be the little-endian unsigned integer denoted by the next + m - 59 bytes. The next 1 + n bytes after that are literal bytes. + +For copy tags, length bytes are copied from offset bytes ago, in the style of +Lempel-Ziv compression algorithms. In particular: + - For l == 1, the offset ranges in [0, 1<<11) and the length in [4, 12). + The length is 4 + the low 3 bits of m. The high 3 bits of m form bits 8-10 + of the offset. The next byte is bits 0-7 of the offset. + - For l == 2, the offset ranges in [0, 1<<16) and the length in [1, 65). + The length is 1 + m. The offset is the little-endian unsigned integer + denoted by the next 2 bytes. + - For l == 3, this tag is a legacy format that is no longer issued by most + encoders. Nonetheless, the offset ranges in [0, 1<<32) and the length in + [1, 65). The length is 1 + m. The offset is the little-endian unsigned + integer denoted by the next 4 bytes. +*/ +const ( + tagLiteral = 0x00 + tagCopy1 = 0x01 + tagCopy2 = 0x02 + tagCopy4 = 0x03 +) + +const ( + checksumSize = 4 + chunkHeaderSize = 4 + magicChunk = "\xff\x06\x00\x00" + magicBody + magicBody = "sNaPpY" + + // maxBlockSize is the maximum size of the input to encodeBlock. It is not + // part of the wire format per se, but some parts of the encoder assume + // that an offset fits into a uint16. + // + // Also, for the framing format (Writer type instead of Encode function), + // https://github.com/google/snappy/blob/master/framing_format.txt says + // that "the uncompressed data in a chunk must be no longer than 65536 + // bytes". + maxBlockSize = 65536 + + // maxEncodedLenOfMaxBlockSize equals MaxEncodedLen(maxBlockSize), but is + // hard coded to be a const instead of a variable, so that obufLen can also + // be a const. Their equivalence is confirmed by + // TestMaxEncodedLenOfMaxBlockSize. + maxEncodedLenOfMaxBlockSize = 76490 + + obufHeaderLen = len(magicChunk) + checksumSize + chunkHeaderSize + obufLen = obufHeaderLen + maxEncodedLenOfMaxBlockSize +) + +const ( + chunkTypeCompressedData = 0x00 + chunkTypeUncompressedData = 0x01 + chunkTypePadding = 0xfe + chunkTypeStreamIdentifier = 0xff +) + +var crcTable = crc32.MakeTable(crc32.Castagnoli) + +// crc implements the checksum specified in section 3 of +// https://github.com/google/snappy/blob/master/framing_format.txt +func crc(b []byte) uint32 { + c := crc32.Update(0, crcTable, b) + return uint32(c>>15|c<<17) + 0xa282ead8 +} diff --git a/vendor/github.com/klauspost/compress/zstd/README.md b/vendor/github.com/klauspost/compress/zstd/README.md new file mode 100644 index 000000000..bc977a302 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/README.md @@ -0,0 +1,393 @@ +# zstd + +[Zstandard](https://facebook.github.io/zstd/) is a real-time compression algorithm, providing high compression ratios. +It offers a very wide range of compression / speed trade-off, while being backed by a very fast decoder. +A high performance compression algorithm is implemented. For now focused on speed. + +This package provides [compression](#Compressor) to and [decompression](#Decompressor) of Zstandard content. +Note that custom dictionaries are not supported yet, so if your code relies on that, +you cannot use the package as-is. + +This package is pure Go and without use of "unsafe". +If a significant speedup can be achieved using "unsafe", it may be added as an option later. + +The `zstd` package is provided as open source software using a Go standard license. + +Currently the package is heavily optimized for 64 bit processors and will be significantly slower on 32 bit processors. + +## Installation + +Install using `go get -u github.com/klauspost/compress`. The package is located in `github.com/klauspost/compress/zstd`. + +Godoc Documentation: https://godoc.org/github.com/klauspost/compress/zstd + + +## Compressor + +### Status: + +STABLE - there may always be subtle bugs, a wide variety of content has been tested and the library is actively +used by several projects. This library is being continuously [fuzz-tested](https://github.com/klauspost/compress-fuzz), +kindly supplied by [fuzzit.dev](https://fuzzit.dev/). + +There may still be specific combinations of data types/size/settings that could lead to edge cases, +so as always, testing is recommended. + +For now, a high speed (fastest) and medium-fast (default) compressor has been implemented. + +The "Fastest" compression ratio is roughly equivalent to zstd level 1. +The "Default" compression ratio is roughly equivalent to zstd level 3 (default). + +In terms of speed, it is typically 2x as fast as the stdlib deflate/gzip in its fastest mode. +The compression ratio compared to stdlib is around level 3, but usually 3x as fast. + +Compared to cgo zstd, the speed is around level 3 (default), but compression slightly worse, between level 1&2. + + +### Usage + +An Encoder can be used for either compressing a stream via the +`io.WriteCloser` interface supported by the Encoder or as multiple independent +tasks via the `EncodeAll` function. +Smaller encodes are encouraged to use the EncodeAll function. +Use `NewWriter` to create a new instance that can be used for both. + +To create a writer with default options, do like this: + +```Go +// Compress input to output. +func Compress(in io.Reader, out io.Writer) error { + w, err := NewWriter(output) + if err != nil { + return err + } + _, err := io.Copy(w, input) + if err != nil { + enc.Close() + return err + } + return enc.Close() +} +``` + +Now you can encode by writing data to `enc`. The output will be finished writing when `Close()` is called. +Even if your encode fails, you should still call `Close()` to release any resources that may be held up. + +The above is fine for big encodes. However, whenever possible try to *reuse* the writer. + +To reuse the encoder, you can use the `Reset(io.Writer)` function to change to another output. +This will allow the encoder to reuse all resources and avoid wasteful allocations. + +Currently stream encoding has 'light' concurrency, meaning up to 2 goroutines can be working on part +of a stream. This is independent of the `WithEncoderConcurrency(n)`, but that is likely to change +in the future. So if you want to limit concurrency for future updates, specify the concurrency +you would like. + +You can specify your desired compression level using `WithEncoderLevel()` option. Currently only pre-defined +compression settings can be specified. + +#### Future Compatibility Guarantees + +This will be an evolving project. When using this package it is important to note that both the compression efficiency and speed may change. + +The goal will be to keep the default efficiency at the default zstd (level 3). +However the encoding should never be assumed to remain the same, +and you should not use hashes of compressed output for similarity checks. + +The Encoder can be assumed to produce the same output from the exact same code version. +However, the may be modes in the future that break this, +although they will not be enabled without an explicit option. + +This encoder is not designed to (and will probably never) output the exact same bitstream as the reference encoder. + +Also note, that the cgo decompressor currently does not [report all errors on invalid input](https://github.com/DataDog/zstd/issues/59), +[omits error checks](https://github.com/DataDog/zstd/issues/61), [ignores checksums](https://github.com/DataDog/zstd/issues/43) +and seems to ignore concatenated streams, even though [it is part of the spec](https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#frames). + +#### Blocks + +For compressing small blocks, the returned encoder has a function called `EncodeAll(src, dst []byte) []byte`. + +`EncodeAll` will encode all input in src and append it to dst. +This function can be called concurrently, but each call will only run on a single goroutine. + +Encoded blocks can be concatenated and the result will be the combined input stream. +Data compressed with EncodeAll can be decoded with the Decoder, using either a stream or `DecodeAll`. + +Especially when encoding blocks you should take special care to reuse the encoder. +This will effectively make it run without allocations after a warmup period. +To make it run completely without allocations, supply a destination buffer with space for all content. + +```Go +import "github.com/klauspost/compress/zstd" + +// Create a writer that caches compressors. +// For this operation type we supply a nil Reader. +var encoder, _ = zstd.NewWriter(nil) + +// Compress a buffer. +// If you have a destination buffer, the allocation in the call can also be eliminated. +func Compress(src []byte) []byte { + return encoder.EncodeAll(src, make([]byte, 0, len(src))) +} +``` + +You can control the maximum number of concurrent encodes using the `WithEncoderConcurrency(n)` +option when creating the writer. + +Using the Encoder for both a stream and individual blocks concurrently is safe. + +### Performance + +I have collected some speed examples to compare speed and compression against other compressors. + +* `file` is the input file. +* `out` is the compressor used. `zskp` is this package. `gzstd` is gzip standard library. `zstd` is the Datadog cgo library. +* `level` is the compression level used. For `zskp` level 1 is "fastest", level 2 is "default". +* `insize`/`outsize` is the input/output size. +* `millis` is the number of milliseconds used for compression. +* `mb/s` is megabytes (2^20 bytes) per second. + +``` +The test data for the Large Text Compression Benchmark is the first +10^9 bytes of the English Wikipedia dump on Mar. 3, 2006. +http://mattmahoney.net/dc/textdata.html + +file out level insize outsize millis mb/s +enwik9 zskp 1 1000000000 343833033 5840 163.30 +enwik9 zskp 2 1000000000 317822183 8449 112.87 +enwik9 gzstd 1 1000000000 382578136 13627 69.98 +enwik9 gzstd 3 1000000000 349139651 22344 42.68 +enwik9 zstd 1 1000000000 357416379 4838 197.12 +enwik9 zstd 3 1000000000 313734522 7556 126.21 + +GOB stream of binary data. Highly compressible. +https://files.klauspost.com/compress/gob-stream.7z + +file out level insize outsize millis mb/s +gob-stream zskp 1 1911399616 234981983 5100 357.42 +gob-stream zskp 2 1911399616 208674003 6698 272.15 +gob-stream gzstd 1 1911399616 357382641 14727 123.78 +gob-stream gzstd 3 1911399616 327835097 17005 107.19 +gob-stream zstd 1 1911399616 250787165 4075 447.22 +gob-stream zstd 3 1911399616 208191888 5511 330.77 + +Highly compressible JSON file. Similar to logs in a lot of ways. +https://files.klauspost.com/compress/adresser.001.gz + +file out level insize outsize millis mb/s +adresser.001 zskp 1 1073741824 18510122 1477 692.83 +adresser.001 zskp 2 1073741824 19831697 1705 600.59 +adresser.001 gzstd 1 1073741824 47755503 3079 332.47 +adresser.001 gzstd 3 1073741824 40052381 3051 335.63 +adresser.001 zstd 1 1073741824 16135896 994 1030.18 +adresser.001 zstd 3 1073741824 17794465 905 1131.49 + +VM Image, Linux mint with a few installed applications: +https://files.klauspost.com/compress/rawstudio-mint14.7z + +file out level insize outsize millis mb/s +rawstudio-mint14.tar zskp 1 8558382592 3648168838 33398 244.38 +rawstudio-mint14.tar zskp 2 8558382592 3376721436 50962 160.16 +rawstudio-mint14.tar gzstd 1 8558382592 3926257486 84712 96.35 +rawstudio-mint14.tar gzstd 3 8558382592 3740711978 176344 46.28 +rawstudio-mint14.tar zstd 1 8558382592 3607859742 27903 292.51 +rawstudio-mint14.tar zstd 3 8558382592 3341710879 46700 174.77 + + +The test data is designed to test archivers in realistic backup scenarios. +http://mattmahoney.net/dc/10gb.html + +file out level insize outsize millis mb/s +10gb.tar zskp 1 10065157632 4883149814 45715 209.97 +10gb.tar zskp 2 10065157632 4638110010 60970 157.44 +10gb.tar gzstd 1 10065157632 5198296126 97769 98.18 +10gb.tar gzstd 3 10065157632 4932665487 313427 30.63 +10gb.tar zstd 1 10065157632 4940796535 40391 237.65 +10gb.tar zstd 3 10065157632 4638618579 52911 181.42 + +Silesia Corpus: +http://sun.aei.polsl.pl/~sdeor/corpus/silesia.zip + +file out level insize outsize millis mb/s +silesia.tar zskp 1 211947520 73025800 1108 182.26 +silesia.tar zskp 2 211947520 67674684 1599 126.41 +silesia.tar gzstd 1 211947520 80007735 2515 80.37 +silesia.tar gzstd 3 211947520 73133380 4259 47.45 +silesia.tar zstd 1 211947520 73513991 933 216.64 +silesia.tar zstd 3 211947520 66793301 1377 146.79 +``` + +### Converters + +As part of the development process a *Snappy* -> *Zstandard* converter was also built. + +This can convert a *framed* [Snappy Stream](https://godoc.org/github.com/golang/snappy#Writer) to a zstd stream. +Note that a single block is not framed. + +Conversion is done by converting the stream directly from Snappy without intermediate full decoding. +Therefore the compression ratio is much less than what can be done by a full decompression +and compression, and a faulty Snappy stream may lead to a faulty Zstandard stream without +any errors being generated. +No CRC value is being generated and not all CRC values of the Snappy stream are checked. +However, it provides really fast re-compression of Snappy streams. + + +``` +BenchmarkSnappy_ConvertSilesia-8 1 1156001600 ns/op 183.35 MB/s +Snappy len 103008711 -> zstd len 82687318 + +BenchmarkSnappy_Enwik9-8 1 6472998400 ns/op 154.49 MB/s +Snappy len 508028601 -> zstd len 390921079 +``` + + +```Go + s := zstd.SnappyConverter{} + n, err = s.Convert(input, output) + if err != nil { + fmt.Println("Re-compressed stream to", n, "bytes") + } +``` + +The converter `s` can be reused to avoid allocations, even after errors. + + +## Decompressor + +Staus: STABLE - there may still be subtle bugs, but a wide variety of content has been tested. + +This library is being continuously [fuzz-tested](https://github.com/klauspost/compress-fuzz), +kindly supplied by [fuzzit.dev](https://fuzzit.dev/). +The main purpose of the fuzz testing is to ensure that it is not possible to crash the decoder, +or run it past its limits with ANY input provided. + +### Usage + +The package has been designed for two main usages, big streams of data and smaller in-memory buffers. +There are two main usages of the package for these. Both of them are accessed by creating a `Decoder`. + +For streaming use a simple setup could look like this: + +```Go +import "github.com/klauspost/compress/zstd" + +func Decompress(in io.Reader, out io.Writer) error { + d, err := zstd.NewReader(input) + if err != nil { + return err + } + defer d.Close() + + // Copy content... + _, err := io.Copy(out, d) + return err +} +``` + +It is important to use the "Close" function when you no longer need the Reader to stop running goroutines. +See "Allocation-less operation" below. + +For decoding buffers, it could look something like this: + +```Go +import "github.com/klauspost/compress/zstd" + +// Create a reader that caches decompressors. +// For this operation type we supply a nil Reader. +var decoder, _ = zstd.NewReader(nil) + +// Decompress a buffer. We don't supply a destination buffer, +// so it will be allocated by the decoder. +func Decompress(src []byte) ([]byte, error) { + return decoder.DecodeAll(src, nil) +} +``` + +Both of these cases should provide the functionality needed. +The decoder can be used for *concurrent* decompression of multiple buffers. +It will only allow a certain number of concurrent operations to run. +To tweak that yourself use the `WithDecoderConcurrency(n)` option when creating the decoder. + +### Allocation-less operation + +The decoder has been designed to operate without allocations after a warmup. + +This means that you should *store* the decoder for best performance. +To re-use a stream decoder, use the `Reset(r io.Reader) error` to switch to another stream. +A decoder can safely be re-used even if the previous stream failed. + +To release the resources, you must call the `Close()` function on a decoder. +After this it can *no longer be reused*, but all running goroutines will be stopped. +So you *must* use this if you will no longer need the Reader. + +For decompressing smaller buffers a single decoder can be used. +When decoding buffers, you can supply a destination slice with length 0 and your expected capacity. +In this case no unneeded allocations should be made. + +### Concurrency + +The buffer decoder does everything on the same goroutine and does nothing concurrently. +It can however decode several buffers concurrently. Use `WithDecoderConcurrency(n)` to limit that. + +The stream decoder operates on + +* One goroutine reads input and splits the input to several block decoders. +* A number of decoders will decode blocks. +* A goroutine coordinates these blocks and sends history from one to the next. + +So effectively this also means the decoder will "read ahead" and prepare data to always be available for output. + +Since "blocks" are quite dependent on the output of the previous block stream decoding will only have limited concurrency. + +In practice this means that concurrency is often limited to utilizing about 2 cores effectively. + + +### Benchmarks + +These are some examples of performance compared to [datadog cgo library](https://github.com/DataDog/zstd). + +The first two are streaming decodes and the last are smaller inputs. + +``` +BenchmarkDecoderSilesia-8 20 642550210 ns/op 329.85 MB/s 3101 B/op 8 allocs/op +BenchmarkDecoderSilesiaCgo-8 100 384930000 ns/op 550.61 MB/s 451878 B/op 9713 allocs/op + +BenchmarkDecoderEnwik9-2 10 3146000080 ns/op 317.86 MB/s 2649 B/op 9 allocs/op +BenchmarkDecoderEnwik9Cgo-2 20 1905900000 ns/op 524.69 MB/s 1125120 B/op 45785 allocs/op + +BenchmarkDecoder_DecodeAll/z000000.zst-8 200 7049994 ns/op 138.26 MB/s 40 B/op 2 allocs/op +BenchmarkDecoder_DecodeAll/z000001.zst-8 100000 19560 ns/op 97.49 MB/s 40 B/op 2 allocs/op +BenchmarkDecoder_DecodeAll/z000002.zst-8 5000 297599 ns/op 236.99 MB/s 40 B/op 2 allocs/op +BenchmarkDecoder_DecodeAll/z000003.zst-8 2000 725502 ns/op 141.17 MB/s 40 B/op 2 allocs/op +BenchmarkDecoder_DecodeAll/z000004.zst-8 200000 9314 ns/op 54.54 MB/s 40 B/op 2 allocs/op +BenchmarkDecoder_DecodeAll/z000005.zst-8 10000 137500 ns/op 104.72 MB/s 40 B/op 2 allocs/op +BenchmarkDecoder_DecodeAll/z000006.zst-8 500 2316009 ns/op 206.06 MB/s 40 B/op 2 allocs/op +BenchmarkDecoder_DecodeAll/z000007.zst-8 20000 64499 ns/op 344.90 MB/s 40 B/op 2 allocs/op +BenchmarkDecoder_DecodeAll/z000008.zst-8 50000 24900 ns/op 219.56 MB/s 40 B/op 2 allocs/op +BenchmarkDecoder_DecodeAll/z000009.zst-8 1000 2348999 ns/op 154.01 MB/s 40 B/op 2 allocs/op + +BenchmarkDecoder_DecodeAllCgo/z000000.zst-8 500 4268005 ns/op 228.38 MB/s 1228849 B/op 3 allocs/op +BenchmarkDecoder_DecodeAllCgo/z000001.zst-8 100000 15250 ns/op 125.05 MB/s 2096 B/op 3 allocs/op +BenchmarkDecoder_DecodeAllCgo/z000002.zst-8 10000 147399 ns/op 478.49 MB/s 73776 B/op 3 allocs/op +BenchmarkDecoder_DecodeAllCgo/z000003.zst-8 5000 320798 ns/op 319.27 MB/s 139312 B/op 3 allocs/op +BenchmarkDecoder_DecodeAllCgo/z000004.zst-8 200000 10004 ns/op 50.77 MB/s 560 B/op 3 allocs/op +BenchmarkDecoder_DecodeAllCgo/z000005.zst-8 20000 73599 ns/op 195.64 MB/s 19120 B/op 3 allocs/op +BenchmarkDecoder_DecodeAllCgo/z000006.zst-8 1000 1119003 ns/op 426.48 MB/s 557104 B/op 3 allocs/op +BenchmarkDecoder_DecodeAllCgo/z000007.zst-8 20000 103450 ns/op 215.04 MB/s 71296 B/op 9 allocs/op +BenchmarkDecoder_DecodeAllCgo/z000008.zst-8 100000 20130 ns/op 271.58 MB/s 6192 B/op 3 allocs/op +BenchmarkDecoder_DecodeAllCgo/z000009.zst-8 2000 1123500 ns/op 322.00 MB/s 368688 B/op 3 allocs/op +``` + +This reflects the performance around May 2019, but this may be out of date. + +# Contributions + +Contributions are always welcome. +For new features/fixes, remember to add tests and for performance enhancements include benchmarks. + +For sending files for reproducing errors use a service like [goobox](https://goobox.io/#/upload) or similar to share your files. + +For general feedback and experience reports, feel free to open an issue or write me on [Twitter](https://twitter.com/sh0dan). + +This package includes the excellent [`github.com/cespare/xxhash`](https://github.com/cespare/xxhash) package Copyright (c) 2016 Caleb Spare. diff --git a/vendor/github.com/klauspost/compress/zstd/bitreader.go b/vendor/github.com/klauspost/compress/zstd/bitreader.go new file mode 100644 index 000000000..15d79d439 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/bitreader.go @@ -0,0 +1,121 @@ +// Copyright 2019+ Klaus Post. All rights reserved. +// License information can be found in the LICENSE file. +// Based on work by Yann Collet, released under BSD License. + +package zstd + +import ( + "errors" + "io" + "math/bits" +) + +// bitReader reads a bitstream in reverse. +// The last set bit indicates the start of the stream and is used +// for aligning the input. +type bitReader struct { + in []byte + off uint // next byte to read is at in[off - 1] + value uint64 // Maybe use [16]byte, but shifting is awkward. + bitsRead uint8 +} + +// init initializes and resets the bit reader. +func (b *bitReader) init(in []byte) error { + if len(in) < 1 { + return errors.New("corrupt stream: too short") + } + b.in = in + b.off = uint(len(in)) + // The highest bit of the last byte indicates where to start + v := in[len(in)-1] + if v == 0 { + return errors.New("corrupt stream, did not find end of stream") + } + b.bitsRead = 64 + b.value = 0 + b.fill() + b.fill() + b.bitsRead += 8 - uint8(highBits(uint32(v))) + return nil +} + +// getBits will return n bits. n can be 0. +func (b *bitReader) getBits(n uint8) int { + if n == 0 /*|| b.bitsRead >= 64 */ { + return 0 + } + return b.getBitsFast(n) +} + +// getBitsFast requires that at least one bit is requested every time. +// There are no checks if the buffer is filled. +func (b *bitReader) getBitsFast(n uint8) int { + const regMask = 64 - 1 + v := uint32((b.value << (b.bitsRead & regMask)) >> ((regMask + 1 - n) & regMask)) + b.bitsRead += n + return int(v) +} + +// fillFast() will make sure at least 32 bits are available. +// There must be at least 4 bytes available. +func (b *bitReader) fillFast() { + if b.bitsRead < 32 { + return + } + // Do single re-slice to avoid bounds checks. + v := b.in[b.off-4 : b.off] + low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) + b.value = (b.value << 32) | uint64(low) + b.bitsRead -= 32 + b.off -= 4 +} + +// fill() will make sure at least 32 bits are available. +func (b *bitReader) fill() { + if b.bitsRead < 32 { + return + } + if b.off >= 4 { + v := b.in[b.off-4 : b.off] + low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) + b.value = (b.value << 32) | uint64(low) + b.bitsRead -= 32 + b.off -= 4 + return + } + for b.off > 0 { + b.value = (b.value << 8) | uint64(b.in[b.off-1]) + b.bitsRead -= 8 + b.off-- + } +} + +// finished returns true if all bits have been read from the bit stream. +func (b *bitReader) finished() bool { + return b.off == 0 && b.bitsRead >= 64 +} + +// overread returns true if more bits have been requested than is on the stream. +func (b *bitReader) overread() bool { + return b.bitsRead > 64 +} + +// remain returns the number of bits remaining. +func (b *bitReader) remain() uint { + return b.off*8 + 64 - uint(b.bitsRead) +} + +// close the bitstream and returns an error if out-of-buffer reads occurred. +func (b *bitReader) close() error { + // Release reference. + b.in = nil + if b.bitsRead > 64 { + return io.ErrUnexpectedEOF + } + return nil +} + +func highBits(val uint32) (n uint32) { + return uint32(bits.Len32(val) - 1) +} diff --git a/vendor/github.com/klauspost/compress/zstd/bitwriter.go b/vendor/github.com/klauspost/compress/zstd/bitwriter.go new file mode 100644 index 000000000..303ae90f9 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/bitwriter.go @@ -0,0 +1,169 @@ +// Copyright 2018 Klaus Post. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// Based on work Copyright (c) 2013, Yann Collet, released under BSD License. + +package zstd + +import "fmt" + +// bitWriter will write bits. +// First bit will be LSB of the first byte of output. +type bitWriter struct { + bitContainer uint64 + nBits uint8 + out []byte +} + +// bitMask16 is bitmasks. Has extra to avoid bounds check. +var bitMask16 = [32]uint16{ + 0, 1, 3, 7, 0xF, 0x1F, + 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, + 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0xFFFF, + 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, + 0xFFFF, 0xFFFF} /* up to 16 bits */ + +var bitMask32 = [32]uint32{ + 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, + 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, + 0x1ffff, 0x3ffff, 0x7FFFF, 0xfFFFF, 0x1fFFFF, 0x3fFFFF, 0x7fFFFF, 0xffFFFF, + 0x1ffFFFF, 0x3ffFFFF, 0x7ffFFFF, 0xfffFFFF, 0x1fffFFFF, 0x3fffFFFF, 0x7fffFFFF, +} // up to 32 bits + +// addBits16NC will add up to 16 bits. +// It will not check if there is space for them, +// so the caller must ensure that it has flushed recently. +func (b *bitWriter) addBits16NC(value uint16, bits uint8) { + b.bitContainer |= uint64(value&bitMask16[bits&31]) << (b.nBits & 63) + b.nBits += bits +} + +// addBits32NC will add up to 32 bits. +// It will not check if there is space for them, +// so the caller must ensure that it has flushed recently. +func (b *bitWriter) addBits32NC(value uint32, bits uint8) { + b.bitContainer |= uint64(value&bitMask32[bits&31]) << (b.nBits & 63) + b.nBits += bits +} + +// addBits16Clean will add up to 16 bits. value may not contain more set bits than indicated. +// It will not check if there is space for them, so the caller must ensure that it has flushed recently. +func (b *bitWriter) addBits16Clean(value uint16, bits uint8) { + b.bitContainer |= uint64(value) << (b.nBits & 63) + b.nBits += bits +} + +// flush will flush all pending full bytes. +// There will be at least 56 bits available for writing when this has been called. +// Using flush32 is faster, but leaves less space for writing. +func (b *bitWriter) flush() { + v := b.nBits >> 3 + switch v { + case 0: + case 1: + b.out = append(b.out, + byte(b.bitContainer), + ) + case 2: + b.out = append(b.out, + byte(b.bitContainer), + byte(b.bitContainer>>8), + ) + case 3: + b.out = append(b.out, + byte(b.bitContainer), + byte(b.bitContainer>>8), + byte(b.bitContainer>>16), + ) + case 4: + b.out = append(b.out, + byte(b.bitContainer), + byte(b.bitContainer>>8), + byte(b.bitContainer>>16), + byte(b.bitContainer>>24), + ) + case 5: + b.out = append(b.out, + byte(b.bitContainer), + byte(b.bitContainer>>8), + byte(b.bitContainer>>16), + byte(b.bitContainer>>24), + byte(b.bitContainer>>32), + ) + case 6: + b.out = append(b.out, + byte(b.bitContainer), + byte(b.bitContainer>>8), + byte(b.bitContainer>>16), + byte(b.bitContainer>>24), + byte(b.bitContainer>>32), + byte(b.bitContainer>>40), + ) + case 7: + b.out = append(b.out, + byte(b.bitContainer), + byte(b.bitContainer>>8), + byte(b.bitContainer>>16), + byte(b.bitContainer>>24), + byte(b.bitContainer>>32), + byte(b.bitContainer>>40), + byte(b.bitContainer>>48), + ) + case 8: + b.out = append(b.out, + byte(b.bitContainer), + byte(b.bitContainer>>8), + byte(b.bitContainer>>16), + byte(b.bitContainer>>24), + byte(b.bitContainer>>32), + byte(b.bitContainer>>40), + byte(b.bitContainer>>48), + byte(b.bitContainer>>56), + ) + default: + panic(fmt.Errorf("bits (%d) > 64", b.nBits)) + } + b.bitContainer >>= v << 3 + b.nBits &= 7 +} + +// flush32 will flush out, so there are at least 32 bits available for writing. +func (b *bitWriter) flush32() { + if b.nBits < 32 { + return + } + b.out = append(b.out, + byte(b.bitContainer), + byte(b.bitContainer>>8), + byte(b.bitContainer>>16), + byte(b.bitContainer>>24)) + b.nBits -= 32 + b.bitContainer >>= 32 +} + +// flushAlign will flush remaining full bytes and align to next byte boundary. +func (b *bitWriter) flushAlign() { + nbBytes := (b.nBits + 7) >> 3 + for i := uint8(0); i < nbBytes; i++ { + b.out = append(b.out, byte(b.bitContainer>>(i*8))) + } + b.nBits = 0 + b.bitContainer = 0 +} + +// close will write the alignment bit and write the final byte(s) +// to the output. +func (b *bitWriter) close() error { + // End mark + b.addBits16Clean(1, 1) + // flush until next byte. + b.flushAlign() + return nil +} + +// reset and continue writing by appending to out. +func (b *bitWriter) reset(out []byte) { + b.bitContainer = 0 + b.nBits = 0 + b.out = out +} diff --git a/vendor/github.com/klauspost/compress/zstd/blockdec.go b/vendor/github.com/klauspost/compress/zstd/blockdec.go new file mode 100644 index 000000000..ed670bcc7 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/blockdec.go @@ -0,0 +1,716 @@ +// Copyright 2019+ Klaus Post. All rights reserved. +// License information can be found in the LICENSE file. +// Based on work by Yann Collet, released under BSD License. + +package zstd + +import ( + "errors" + "fmt" + "io" + "sync" + + "github.com/klauspost/compress/huff0" + "github.com/klauspost/compress/zstd/internal/xxhash" +) + +type blockType uint8 + +//go:generate stringer -type=blockType,literalsBlockType,seqCompMode,tableIndex + +const ( + blockTypeRaw blockType = iota + blockTypeRLE + blockTypeCompressed + blockTypeReserved +) + +type literalsBlockType uint8 + +const ( + literalsBlockRaw literalsBlockType = iota + literalsBlockRLE + literalsBlockCompressed + literalsBlockTreeless +) + +const ( + // maxCompressedBlockSize is the biggest allowed compressed block size (128KB) + maxCompressedBlockSize = 128 << 10 + + // Maximum possible block size (all Raw+Uncompressed). + maxBlockSize = (1 << 21) - 1 + + // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#literals_section_header + maxCompressedLiteralSize = 1 << 18 + maxRLELiteralSize = 1 << 20 + maxMatchLen = 131074 + maxSequences = 0x7f00 + 0xffff + + // We support slightly less than the reference decoder to be able to + // use ints on 32 bit archs. + maxOffsetBits = 30 +) + +var ( + huffDecoderPool = sync.Pool{New: func() interface{} { + return &huff0.Scratch{} + }} + + fseDecoderPool = sync.Pool{New: func() interface{} { + return &fseDecoder{} + }} +) + +type blockDec struct { + // Raw source data of the block. + data []byte + dataStorage []byte + + // Destination of the decoded data. + dst []byte + + // Buffer for literals data. + literalBuf []byte + + // Window size of the block. + WindowSize uint64 + Type blockType + RLESize uint32 + + // Is this the last block of a frame? + Last bool + + // Use less memory + lowMem bool + history chan *history + input chan struct{} + result chan decodeOutput + sequenceBuf []seq + tmp [4]byte + err error + decWG sync.WaitGroup +} + +func (b *blockDec) String() string { + if b == nil { + return "" + } + return fmt.Sprintf("Steam Size: %d, Type: %v, Last: %t, Window: %d", len(b.data), b.Type, b.Last, b.WindowSize) +} + +func newBlockDec(lowMem bool) *blockDec { + b := blockDec{ + lowMem: lowMem, + result: make(chan decodeOutput, 1), + input: make(chan struct{}, 1), + history: make(chan *history, 1), + } + b.decWG.Add(1) + go b.startDecoder() + return &b +} + +// reset will reset the block. +// Input must be a start of a block and will be at the end of the block when returned. +func (b *blockDec) reset(br byteBuffer, windowSize uint64) error { + b.WindowSize = windowSize + tmp := br.readSmall(3) + if tmp == nil { + if debug { + println("Reading block header:", io.ErrUnexpectedEOF) + } + return io.ErrUnexpectedEOF + } + bh := uint32(tmp[0]) | (uint32(tmp[1]) << 8) | (uint32(tmp[2]) << 16) + b.Last = bh&1 != 0 + b.Type = blockType((bh >> 1) & 3) + // find size. + cSize := int(bh >> 3) + switch b.Type { + case blockTypeReserved: + return ErrReservedBlockType + case blockTypeRLE: + b.RLESize = uint32(cSize) + cSize = 1 + case blockTypeCompressed: + if debug { + println("Data size on stream:", cSize) + } + b.RLESize = 0 + if cSize > maxCompressedBlockSize || uint64(cSize) > b.WindowSize { + if debug { + printf("compressed block too big: csize:%d block: %+v\n", uint64(cSize), b) + } + return ErrCompressedSizeTooBig + } + default: + b.RLESize = 0 + } + + // Read block data. + if cap(b.dataStorage) < cSize { + if b.lowMem { + b.dataStorage = make([]byte, 0, cSize) + } else { + b.dataStorage = make([]byte, 0, maxBlockSize) + } + } + if cap(b.dst) <= maxBlockSize { + b.dst = make([]byte, 0, maxBlockSize+1) + } + var err error + b.data, err = br.readBig(cSize, b.dataStorage) + if err != nil { + if debug { + println("Reading block:", err, "(", cSize, ")", len(b.data)) + printf("%T", br) + } + return err + } + return nil +} + +// sendEOF will make the decoder send EOF on this frame. +func (b *blockDec) sendErr(err error) { + b.Last = true + b.Type = blockTypeReserved + b.err = err + b.input <- struct{}{} +} + +// Close will release resources. +// Closed blockDec cannot be reset. +func (b *blockDec) Close() { + close(b.input) + close(b.history) + close(b.result) + b.decWG.Wait() +} + +// decodeAsync will prepare decoding the block when it receives input. +// This will separate output and history. +func (b *blockDec) startDecoder() { + defer b.decWG.Done() + for range b.input { + //println("blockDec: Got block input") + switch b.Type { + case blockTypeRLE: + if cap(b.dst) < int(b.RLESize) { + if b.lowMem { + b.dst = make([]byte, b.RLESize) + } else { + b.dst = make([]byte, maxBlockSize) + } + } + o := decodeOutput{ + d: b, + b: b.dst[:b.RLESize], + err: nil, + } + v := b.data[0] + for i := range o.b { + o.b[i] = v + } + hist := <-b.history + hist.append(o.b) + b.result <- o + case blockTypeRaw: + o := decodeOutput{ + d: b, + b: b.data, + err: nil, + } + hist := <-b.history + hist.append(o.b) + b.result <- o + case blockTypeCompressed: + b.dst = b.dst[:0] + err := b.decodeCompressed(nil) + o := decodeOutput{ + d: b, + b: b.dst, + err: err, + } + if debug { + println("Decompressed to", len(b.dst), "bytes, error:", err) + } + b.result <- o + case blockTypeReserved: + // Used for returning errors. + <-b.history + b.result <- decodeOutput{ + d: b, + b: nil, + err: b.err, + } + default: + panic("Invalid block type") + } + if debug { + println("blockDec: Finished block") + } + } +} + +// decodeAsync will prepare decoding the block when it receives the history. +// If history is provided, it will not fetch it from the channel. +func (b *blockDec) decodeBuf(hist *history) error { + switch b.Type { + case blockTypeRLE: + if cap(b.dst) < int(b.RLESize) { + if b.lowMem { + b.dst = make([]byte, b.RLESize) + } else { + b.dst = make([]byte, maxBlockSize) + } + } + b.dst = b.dst[:b.RLESize] + v := b.data[0] + for i := range b.dst { + b.dst[i] = v + } + hist.appendKeep(b.dst) + return nil + case blockTypeRaw: + hist.appendKeep(b.data) + return nil + case blockTypeCompressed: + saved := b.dst + b.dst = hist.b + hist.b = nil + err := b.decodeCompressed(hist) + if debug { + println("Decompressed to total", len(b.dst), "bytes, hash:", xxhash.Sum64(b.dst), "error:", err) + } + hist.b = b.dst + b.dst = saved + return err + case blockTypeReserved: + // Used for returning errors. + return b.err + default: + panic("Invalid block type") + } +} + +// decodeCompressed will start decompressing a block. +// If no history is supplied the decoder will decodeAsync as much as possible +// before fetching from blockDec.history +func (b *blockDec) decodeCompressed(hist *history) error { + in := b.data + delayedHistory := hist == nil + + if delayedHistory { + // We must always grab history. + defer func() { + if hist == nil { + <-b.history + } + }() + } + // There must be at least one byte for Literals_Block_Type and one for Sequences_Section_Header + if len(in) < 2 { + return ErrBlockTooSmall + } + litType := literalsBlockType(in[0] & 3) + var litRegenSize int + var litCompSize int + sizeFormat := (in[0] >> 2) & 3 + var fourStreams bool + switch litType { + case literalsBlockRaw, literalsBlockRLE: + switch sizeFormat { + case 0, 2: + // Regenerated_Size uses 5 bits (0-31). Literals_Section_Header uses 1 byte. + litRegenSize = int(in[0] >> 3) + in = in[1:] + case 1: + // Regenerated_Size uses 12 bits (0-4095). Literals_Section_Header uses 2 bytes. + litRegenSize = int(in[0]>>4) + (int(in[1]) << 4) + in = in[2:] + case 3: + // Regenerated_Size uses 20 bits (0-1048575). Literals_Section_Header uses 3 bytes. + if len(in) < 3 { + println("too small: litType:", litType, " sizeFormat", sizeFormat, len(in)) + return ErrBlockTooSmall + } + litRegenSize = int(in[0]>>4) + (int(in[1]) << 4) + (int(in[2]) << 12) + in = in[3:] + } + case literalsBlockCompressed, literalsBlockTreeless: + switch sizeFormat { + case 0, 1: + // Both Regenerated_Size and Compressed_Size use 10 bits (0-1023). + if len(in) < 3 { + println("too small: litType:", litType, " sizeFormat", sizeFormat, len(in)) + return ErrBlockTooSmall + } + n := uint64(in[0]>>4) + (uint64(in[1]) << 4) + (uint64(in[2]) << 12) + litRegenSize = int(n & 1023) + litCompSize = int(n >> 10) + fourStreams = sizeFormat == 1 + in = in[3:] + case 2: + fourStreams = true + if len(in) < 4 { + println("too small: litType:", litType, " sizeFormat", sizeFormat, len(in)) + return ErrBlockTooSmall + } + n := uint64(in[0]>>4) + (uint64(in[1]) << 4) + (uint64(in[2]) << 12) + (uint64(in[3]) << 20) + litRegenSize = int(n & 16383) + litCompSize = int(n >> 14) + in = in[4:] + case 3: + fourStreams = true + if len(in) < 5 { + println("too small: litType:", litType, " sizeFormat", sizeFormat, len(in)) + return ErrBlockTooSmall + } + n := uint64(in[0]>>4) + (uint64(in[1]) << 4) + (uint64(in[2]) << 12) + (uint64(in[3]) << 20) + (uint64(in[4]) << 28) + litRegenSize = int(n & 262143) + litCompSize = int(n >> 18) + in = in[5:] + } + } + if debug { + println("literals type:", litType, "litRegenSize:", litRegenSize, "litCompSize:", litCompSize, "sizeFormat:", sizeFormat, "4X:", fourStreams) + } + var literals []byte + var huff *huff0.Scratch + switch litType { + case literalsBlockRaw: + if len(in) < litRegenSize { + println("too small: litType:", litType, " sizeFormat", sizeFormat, "remain:", len(in), "want:", litRegenSize) + return ErrBlockTooSmall + } + literals = in[:litRegenSize] + in = in[litRegenSize:] + //printf("Found %d uncompressed literals\n", litRegenSize) + case literalsBlockRLE: + if len(in) < 1 { + println("too small: litType:", litType, " sizeFormat", sizeFormat, "remain:", len(in), "want:", 1) + return ErrBlockTooSmall + } + if cap(b.literalBuf) < litRegenSize { + if b.lowMem { + b.literalBuf = make([]byte, litRegenSize) + } else { + if litRegenSize > maxCompressedLiteralSize { + // Exceptional + b.literalBuf = make([]byte, litRegenSize) + } else { + b.literalBuf = make([]byte, litRegenSize, maxCompressedLiteralSize) + + } + } + } + literals = b.literalBuf[:litRegenSize] + v := in[0] + for i := range literals { + literals[i] = v + } + in = in[1:] + if debug { + printf("Found %d RLE compressed literals\n", litRegenSize) + } + case literalsBlockTreeless: + if len(in) < litCompSize { + println("too small: litType:", litType, " sizeFormat", sizeFormat, "remain:", len(in), "want:", litCompSize) + return ErrBlockTooSmall + } + // Store compressed literals, so we defer decoding until we get history. + literals = in[:litCompSize] + in = in[litCompSize:] + if debug { + printf("Found %d compressed literals\n", litCompSize) + } + case literalsBlockCompressed: + if len(in) < litCompSize { + println("too small: litType:", litType, " sizeFormat", sizeFormat, "remain:", len(in), "want:", litCompSize) + return ErrBlockTooSmall + } + literals = in[:litCompSize] + in = in[litCompSize:] + huff = huffDecoderPool.Get().(*huff0.Scratch) + var err error + // Ensure we have space to store it. + if cap(b.literalBuf) < litRegenSize { + if b.lowMem { + b.literalBuf = make([]byte, 0, litRegenSize) + } else { + b.literalBuf = make([]byte, 0, maxCompressedLiteralSize) + } + } + if huff == nil { + huff = &huff0.Scratch{} + } + huff.Out = b.literalBuf[:0] + huff, literals, err = huff0.ReadTable(literals, huff) + if err != nil { + println("reading huffman table:", err) + return err + } + // Use our out buffer. + huff.Out = b.literalBuf[:0] + huff.MaxDecodedSize = litRegenSize + if fourStreams { + literals, err = huff.Decompress4X(literals, litRegenSize) + } else { + literals, err = huff.Decompress1X(literals) + } + if err != nil { + println("decoding compressed literals:", err) + return err + } + // Make sure we don't leak our literals buffer + huff.Out = nil + if len(literals) != litRegenSize { + return fmt.Errorf("literal output size mismatch want %d, got %d", litRegenSize, len(literals)) + } + if debug { + printf("Decompressed %d literals into %d bytes\n", litCompSize, litRegenSize) + } + } + + // Decode Sequences + // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#sequences-section + if len(in) < 1 { + return ErrBlockTooSmall + } + seqHeader := in[0] + nSeqs := 0 + switch { + case seqHeader == 0: + in = in[1:] + case seqHeader < 128: + nSeqs = int(seqHeader) + in = in[1:] + case seqHeader < 255: + if len(in) < 2 { + return ErrBlockTooSmall + } + nSeqs = int(seqHeader-128)<<8 | int(in[1]) + in = in[2:] + case seqHeader == 255: + if len(in) < 3 { + return ErrBlockTooSmall + } + nSeqs = 0x7f00 + int(in[1]) + (int(in[2]) << 8) + in = in[3:] + } + // Allocate sequences + if cap(b.sequenceBuf) < nSeqs { + if b.lowMem { + b.sequenceBuf = make([]seq, nSeqs) + } else { + // Allocate max + b.sequenceBuf = make([]seq, nSeqs, maxSequences) + } + } else { + // Reuse buffer + b.sequenceBuf = b.sequenceBuf[:nSeqs] + } + var seqs = &sequenceDecs{} + if nSeqs > 0 { + if len(in) < 1 { + return ErrBlockTooSmall + } + br := byteReader{b: in, off: 0} + compMode := br.Uint8() + br.advance(1) + if debug { + printf("Compression modes: 0b%b", compMode) + } + for i := uint(0); i < 3; i++ { + mode := seqCompMode((compMode >> (6 - i*2)) & 3) + if debug { + println("Table", tableIndex(i), "is", mode) + } + var seq *sequenceDec + switch tableIndex(i) { + case tableLiteralLengths: + seq = &seqs.litLengths + case tableOffsets: + seq = &seqs.offsets + case tableMatchLengths: + seq = &seqs.matchLengths + default: + panic("unknown table") + } + switch mode { + case compModePredefined: + seq.fse = &fsePredef[i] + case compModeRLE: + if br.remain() < 1 { + return ErrBlockTooSmall + } + v := br.Uint8() + br.advance(1) + dec := fseDecoderPool.Get().(*fseDecoder) + symb, err := decSymbolValue(v, symbolTableX[i]) + if err != nil { + printf("RLE Transform table (%v) error: %v", tableIndex(i), err) + return err + } + dec.setRLE(symb) + seq.fse = dec + if debug { + printf("RLE set to %+v, code: %v", symb, v) + } + case compModeFSE: + println("Reading table for", tableIndex(i)) + dec := fseDecoderPool.Get().(*fseDecoder) + err := dec.readNCount(&br, uint16(maxTableSymbol[i])) + if err != nil { + println("Read table error:", err) + return err + } + err = dec.transform(symbolTableX[i]) + if err != nil { + println("Transform table error:", err) + return err + } + if debug { + println("Read table ok", "symbolLen:", dec.symbolLen) + } + seq.fse = dec + case compModeRepeat: + seq.repeat = true + } + if br.overread() { + return io.ErrUnexpectedEOF + } + } + in = br.unread() + } + + // Wait for history. + // All time spent after this is critical since it is strictly sequential. + if hist == nil { + hist = <-b.history + if hist.error { + return ErrDecoderClosed + } + } + + // Decode treeless literal block. + if litType == literalsBlockTreeless { + // TODO: We could send the history early WITHOUT the stream history. + // This would allow decoding treeless literials before the byte history is available. + // Silencia stats: Treeless 4393, with: 32775, total: 37168, 11% treeless. + // So not much obvious gain here. + + if hist.huffTree == nil { + return errors.New("literal block was treeless, but no history was defined") + } + // Ensure we have space to store it. + if cap(b.literalBuf) < litRegenSize { + if b.lowMem { + b.literalBuf = make([]byte, 0, litRegenSize) + } else { + b.literalBuf = make([]byte, 0, maxCompressedLiteralSize) + } + } + var err error + // Use our out buffer. + huff = hist.huffTree + huff.Out = b.literalBuf[:0] + huff.MaxDecodedSize = litRegenSize + if fourStreams { + literals, err = huff.Decompress4X(literals, litRegenSize) + } else { + literals, err = huff.Decompress1X(literals) + } + // Make sure we don't leak our literals buffer + huff.Out = nil + if err != nil { + println("decompressing literals:", err) + return err + } + if len(literals) != litRegenSize { + return fmt.Errorf("literal output size mismatch want %d, got %d", litRegenSize, len(literals)) + } + } else { + if hist.huffTree != nil && huff != nil { + huffDecoderPool.Put(hist.huffTree) + hist.huffTree = nil + } + } + if huff != nil { + huff.Out = nil + hist.huffTree = huff + } + if debug { + println("Final literals:", len(literals), "hash:", xxhash.Sum64(literals), "and", nSeqs, "sequences.") + } + + if nSeqs == 0 { + // Decompressed content is defined entirely as Literals Section content. + b.dst = append(b.dst, literals...) + if delayedHistory { + hist.append(literals) + } + return nil + } + + seqs, err := seqs.mergeHistory(&hist.decoders) + if err != nil { + return err + } + if debug { + println("History merged ok") + } + br := &bitReader{} + if err := br.init(in); err != nil { + return err + } + + // TODO: Investigate if sending history without decoders are faster. + // This would allow the sequences to be decoded async and only have to construct stream history. + // If only recent offsets were not transferred, this would be an obvious win. + // Also, if first 3 sequences don't reference recent offsets, all sequences can be decoded. + + if err := seqs.initialize(br, hist, literals, b.dst); err != nil { + println("initializing sequences:", err) + return err + } + + err = seqs.decode(nSeqs, br, hist.b) + if err != nil { + return err + } + if !br.finished() { + return fmt.Errorf("%d extra bits on block, should be 0", br.remain()) + } + + err = br.close() + if err != nil { + printf("Closing sequences: %v, %+v\n", err, *br) + } + if len(b.data) > maxCompressedBlockSize { + return fmt.Errorf("compressed block size too large (%d)", len(b.data)) + } + // Set output and release references. + b.dst = seqs.out + seqs.out, seqs.literals, seqs.hist = nil, nil, nil + + if !delayedHistory { + // If we don't have delayed history, no need to update. + hist.recentOffsets = seqs.prevOffset + return nil + } + if b.Last { + // if last block we don't care about history. + println("Last block, no history returned") + hist.b = hist.b[:0] + return nil + } + hist.append(b.dst) + hist.recentOffsets = seqs.prevOffset + if debug { + println("Finished block with literals:", len(literals), "and", nSeqs, "sequences.") + } + + return nil +} diff --git a/vendor/github.com/klauspost/compress/zstd/blockenc.go b/vendor/github.com/klauspost/compress/zstd/blockenc.go new file mode 100644 index 000000000..04c6b638e --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/blockenc.go @@ -0,0 +1,845 @@ +// Copyright 2019+ Klaus Post. All rights reserved. +// License information can be found in the LICENSE file. +// Based on work by Yann Collet, released under BSD License. + +package zstd + +import ( + "errors" + "fmt" + "math" + "math/bits" + + "github.com/klauspost/compress/huff0" +) + +type blockEnc struct { + size int + literals []byte + sequences []seq + coders seqCoders + litEnc *huff0.Scratch + wr bitWriter + + extraLits int + last bool + + output []byte + recentOffsets [3]uint32 + prevRecentOffsets [3]uint32 +} + +// init should be used once the block has been created. +// If called more than once, the effect is the same as calling reset. +func (b *blockEnc) init() { + if cap(b.literals) < maxCompressedLiteralSize { + b.literals = make([]byte, 0, maxCompressedLiteralSize) + } + const defSeqs = 200 + b.literals = b.literals[:0] + if cap(b.sequences) < defSeqs { + b.sequences = make([]seq, 0, defSeqs) + } + if cap(b.output) < maxCompressedBlockSize { + b.output = make([]byte, 0, maxCompressedBlockSize) + } + if b.coders.mlEnc == nil { + b.coders.mlEnc = &fseEncoder{} + b.coders.mlPrev = &fseEncoder{} + b.coders.ofEnc = &fseEncoder{} + b.coders.ofPrev = &fseEncoder{} + b.coders.llEnc = &fseEncoder{} + b.coders.llPrev = &fseEncoder{} + } + b.litEnc = &huff0.Scratch{WantLogLess: 4} + b.reset(nil) +} + +// initNewEncode can be used to reset offsets and encoders to the initial state. +func (b *blockEnc) initNewEncode() { + b.recentOffsets = [3]uint32{1, 4, 8} + b.litEnc.Reuse = huff0.ReusePolicyNone + b.coders.setPrev(nil, nil, nil) +} + +// reset will reset the block for a new encode, but in the same stream, +// meaning that state will be carried over, but the block content is reset. +// If a previous block is provided, the recent offsets are carried over. +func (b *blockEnc) reset(prev *blockEnc) { + b.extraLits = 0 + b.literals = b.literals[:0] + b.size = 0 + b.sequences = b.sequences[:0] + b.output = b.output[:0] + b.last = false + if prev != nil { + b.recentOffsets = prev.prevRecentOffsets + } +} + +// reset will reset the block for a new encode, but in the same stream, +// meaning that state will be carried over, but the block content is reset. +// If a previous block is provided, the recent offsets are carried over. +func (b *blockEnc) swapEncoders(prev *blockEnc) { + b.coders.swap(&prev.coders) + b.litEnc, prev.litEnc = prev.litEnc, b.litEnc +} + +// blockHeader contains the information for a block header. +type blockHeader uint32 + +// setLast sets the 'last' indicator on a block. +func (h *blockHeader) setLast(b bool) { + if b { + *h = *h | 1 + } else { + const mask = (1 << 24) - 2 + *h = *h & mask + } +} + +// setSize will store the compressed size of a block. +func (h *blockHeader) setSize(v uint32) { + const mask = 7 + *h = (*h)&mask | blockHeader(v<<3) +} + +// setType sets the block type. +func (h *blockHeader) setType(t blockType) { + const mask = 1 | (((1 << 24) - 1) ^ 7) + *h = (*h & mask) | blockHeader(t<<1) +} + +// appendTo will append the block header to a slice. +func (h blockHeader) appendTo(b []byte) []byte { + return append(b, uint8(h), uint8(h>>8), uint8(h>>16)) +} + +// String returns a string representation of the block. +func (h blockHeader) String() string { + return fmt.Sprintf("Type: %d, Size: %d, Last:%t", (h>>1)&3, h>>3, h&1 == 1) +} + +// literalsHeader contains literals header information. +type literalsHeader uint64 + +// setType can be used to set the type of literal block. +func (h *literalsHeader) setType(t literalsBlockType) { + const mask = math.MaxUint64 - 3 + *h = (*h & mask) | literalsHeader(t) +} + +// setSize can be used to set a single size, for uncompressed and RLE content. +func (h *literalsHeader) setSize(regenLen int) { + inBits := bits.Len32(uint32(regenLen)) + // Only retain 2 bits + const mask = 3 + lh := uint64(*h & mask) + switch { + case inBits < 5: + lh |= (uint64(regenLen) << 3) | (1 << 60) + if debug { + got := int(lh>>3) & 0xff + if got != regenLen { + panic(fmt.Sprint("litRegenSize = ", regenLen, "(want) != ", got, "(got)")) + } + } + case inBits < 12: + lh |= (1 << 2) | (uint64(regenLen) << 4) | (2 << 60) + case inBits < 20: + lh |= (3 << 2) | (uint64(regenLen) << 4) | (3 << 60) + default: + panic(fmt.Errorf("internal error: block too big (%d)", regenLen)) + } + *h = literalsHeader(lh) +} + +// setSizes will set the size of a compressed literals section and the input length. +func (h *literalsHeader) setSizes(compLen, inLen int, single bool) { + compBits, inBits := bits.Len32(uint32(compLen)), bits.Len32(uint32(inLen)) + // Only retain 2 bits + const mask = 3 + lh := uint64(*h & mask) + switch { + case compBits <= 10 && inBits <= 10: + if !single { + lh |= 1 << 2 + } + lh |= (uint64(inLen) << 4) | (uint64(compLen) << (10 + 4)) | (3 << 60) + if debug { + const mmask = (1 << 24) - 1 + n := (lh >> 4) & mmask + if int(n&1023) != inLen { + panic(fmt.Sprint("regensize:", int(n&1023), "!=", inLen, inBits)) + } + if int(n>>10) != compLen { + panic(fmt.Sprint("compsize:", int(n>>10), "!=", compLen, compBits)) + } + } + case compBits <= 14 && inBits <= 14: + lh |= (2 << 2) | (uint64(inLen) << 4) | (uint64(compLen) << (14 + 4)) | (4 << 60) + if single { + panic("single stream used with more than 10 bits length.") + } + case compBits <= 18 && inBits <= 18: + lh |= (3 << 2) | (uint64(inLen) << 4) | (uint64(compLen) << (18 + 4)) | (5 << 60) + if single { + panic("single stream used with more than 10 bits length.") + } + default: + panic("internal error: block too big") + } + *h = literalsHeader(lh) +} + +// appendTo will append the literals header to a byte slice. +func (h literalsHeader) appendTo(b []byte) []byte { + size := uint8(h >> 60) + switch size { + case 1: + b = append(b, uint8(h)) + case 2: + b = append(b, uint8(h), uint8(h>>8)) + case 3: + b = append(b, uint8(h), uint8(h>>8), uint8(h>>16)) + case 4: + b = append(b, uint8(h), uint8(h>>8), uint8(h>>16), uint8(h>>24)) + case 5: + b = append(b, uint8(h), uint8(h>>8), uint8(h>>16), uint8(h>>24), uint8(h>>32)) + default: + panic(fmt.Errorf("internal error: literalsHeader has invalid size (%d)", size)) + } + return b +} + +// size returns the output size with currently set values. +func (h literalsHeader) size() int { + return int(h >> 60) +} + +func (h literalsHeader) String() string { + return fmt.Sprintf("Type: %d, SizeFormat: %d, Size: 0x%d, Bytes:%d", literalsBlockType(h&3), (h>>2)&3, h&((1<<60)-1)>>4, h>>60) +} + +// pushOffsets will push the recent offsets to the backup store. +func (b *blockEnc) pushOffsets() { + b.prevRecentOffsets = b.recentOffsets +} + +// pushOffsets will push the recent offsets to the backup store. +func (b *blockEnc) popOffsets() { + b.recentOffsets = b.prevRecentOffsets +} + +// matchOffset will adjust recent offsets and return the adjusted one, +// if it matches a previous offset. +func (b *blockEnc) matchOffset(offset, lits uint32) uint32 { + // Check if offset is one of the recent offsets. + // Adjusts the output offset accordingly. + // Gives a tiny bit of compression, typically around 1%. + if true { + if lits > 0 { + switch offset { + case b.recentOffsets[0]: + offset = 1 + case b.recentOffsets[1]: + b.recentOffsets[1] = b.recentOffsets[0] + b.recentOffsets[0] = offset + offset = 2 + case b.recentOffsets[2]: + b.recentOffsets[2] = b.recentOffsets[1] + b.recentOffsets[1] = b.recentOffsets[0] + b.recentOffsets[0] = offset + offset = 3 + default: + b.recentOffsets[2] = b.recentOffsets[1] + b.recentOffsets[1] = b.recentOffsets[0] + b.recentOffsets[0] = offset + offset += 3 + } + } else { + switch offset { + case b.recentOffsets[1]: + b.recentOffsets[1] = b.recentOffsets[0] + b.recentOffsets[0] = offset + offset = 1 + case b.recentOffsets[2]: + b.recentOffsets[2] = b.recentOffsets[1] + b.recentOffsets[1] = b.recentOffsets[0] + b.recentOffsets[0] = offset + offset = 2 + case b.recentOffsets[0] - 1: + b.recentOffsets[2] = b.recentOffsets[1] + b.recentOffsets[1] = b.recentOffsets[0] + b.recentOffsets[0] = offset + offset = 3 + default: + b.recentOffsets[2] = b.recentOffsets[1] + b.recentOffsets[1] = b.recentOffsets[0] + b.recentOffsets[0] = offset + offset += 3 + } + } + } else { + offset += 3 + } + return offset +} + +// encodeRaw can be used to set the output to a raw representation of supplied bytes. +func (b *blockEnc) encodeRaw(a []byte) { + var bh blockHeader + bh.setLast(b.last) + bh.setSize(uint32(len(a))) + bh.setType(blockTypeRaw) + b.output = bh.appendTo(b.output[:0]) + b.output = append(b.output, a...) + if debug { + println("Adding RAW block, length", len(a)) + } +} + +// encodeRaw can be used to set the output to a raw representation of supplied bytes. +func (b *blockEnc) encodeRawTo(dst, src []byte) []byte { + var bh blockHeader + bh.setLast(b.last) + bh.setSize(uint32(len(src))) + bh.setType(blockTypeRaw) + dst = bh.appendTo(dst) + dst = append(dst, src...) + if debug { + println("Adding RAW block, length", len(src)) + } + return dst +} + +// encodeLits can be used if the block is only litLen. +func (b *blockEnc) encodeLits(raw bool) error { + var bh blockHeader + bh.setLast(b.last) + bh.setSize(uint32(len(b.literals))) + + // Don't compress extremely small blocks + if len(b.literals) < 32 || raw { + if debug { + println("Adding RAW block, length", len(b.literals)) + } + bh.setType(blockTypeRaw) + b.output = bh.appendTo(b.output) + b.output = append(b.output, b.literals...) + return nil + } + + var ( + out []byte + reUsed, single bool + err error + ) + if len(b.literals) >= 1024 { + // Use 4 Streams. + out, reUsed, err = huff0.Compress4X(b.literals, b.litEnc) + if len(out) > len(b.literals)-len(b.literals)>>4 { + // Bail out of compression is too little. + err = huff0.ErrIncompressible + } + } else if len(b.literals) > 32 { + // Use 1 stream + single = true + out, reUsed, err = huff0.Compress1X(b.literals, b.litEnc) + if len(out) > len(b.literals)-len(b.literals)>>4 { + // Bail out of compression is too little. + err = huff0.ErrIncompressible + } + } else { + err = huff0.ErrIncompressible + } + + switch err { + case huff0.ErrIncompressible: + if debug { + println("Adding RAW block, length", len(b.literals)) + } + bh.setType(blockTypeRaw) + b.output = bh.appendTo(b.output) + b.output = append(b.output, b.literals...) + return nil + case huff0.ErrUseRLE: + if debug { + println("Adding RLE block, length", len(b.literals)) + } + bh.setType(blockTypeRLE) + b.output = bh.appendTo(b.output) + b.output = append(b.output, b.literals[0]) + return nil + default: + return err + case nil: + } + // Compressed... + // Now, allow reuse + b.litEnc.Reuse = huff0.ReusePolicyAllow + bh.setType(blockTypeCompressed) + var lh literalsHeader + if reUsed { + if debug { + println("Reused tree, compressed to", len(out)) + } + lh.setType(literalsBlockTreeless) + } else { + if debug { + println("New tree, compressed to", len(out), "tree size:", len(b.litEnc.OutTable)) + } + lh.setType(literalsBlockCompressed) + } + // Set sizes + lh.setSizes(len(out), len(b.literals), single) + bh.setSize(uint32(len(out) + lh.size() + 1)) + + // Write block headers. + b.output = bh.appendTo(b.output) + b.output = lh.appendTo(b.output) + // Add compressed data. + b.output = append(b.output, out...) + // No sequences. + b.output = append(b.output, 0) + return nil +} + +// fuzzFseEncoder can be used to fuzz the FSE encoder. +func fuzzFseEncoder(data []byte) int { + if len(data) > maxSequences || len(data) < 2 { + return 0 + } + enc := fseEncoder{} + hist := enc.Histogram()[:256] + maxSym := uint8(0) + for i, v := range data { + v = v & 63 + data[i] = v + hist[v]++ + if v > maxSym { + maxSym = v + } + } + if maxSym == 0 { + // All 0 + return 0 + } + maxCount := func(a []uint32) int { + var max uint32 + for _, v := range a { + if v > max { + max = v + } + } + return int(max) + } + cnt := maxCount(hist[:maxSym]) + if cnt == len(data) { + // RLE + return 0 + } + enc.HistogramFinished(maxSym, cnt) + err := enc.normalizeCount(len(data)) + if err != nil { + return 0 + } + _, err = enc.writeCount(nil) + if err != nil { + panic(err) + } + return 1 +} + +// encode will encode the block and append the output in b.output. +func (b *blockEnc) encode(raw bool) error { + if len(b.sequences) == 0 { + return b.encodeLits(raw) + } + // We want some difference + if len(b.literals) > (b.size - (b.size >> 5)) { + return errIncompressible + } + + var bh blockHeader + var lh literalsHeader + bh.setLast(b.last) + bh.setType(blockTypeCompressed) + // Store offset of the block header. Needed when we know the size. + bhOffset := len(b.output) + b.output = bh.appendTo(b.output) + + var ( + out []byte + reUsed, single bool + err error + ) + if len(b.literals) >= 1024 && !raw { + // Use 4 Streams. + out, reUsed, err = huff0.Compress4X(b.literals, b.litEnc) + } else if len(b.literals) > 32 && !raw { + // Use 1 stream + single = true + out, reUsed, err = huff0.Compress1X(b.literals, b.litEnc) + } else { + err = huff0.ErrIncompressible + } + + switch err { + case huff0.ErrIncompressible: + lh.setType(literalsBlockRaw) + lh.setSize(len(b.literals)) + b.output = lh.appendTo(b.output) + b.output = append(b.output, b.literals...) + if debug { + println("Adding literals RAW, length", len(b.literals)) + } + case huff0.ErrUseRLE: + lh.setType(literalsBlockRLE) + lh.setSize(len(b.literals)) + b.output = lh.appendTo(b.output) + b.output = append(b.output, b.literals[0]) + if debug { + println("Adding literals RLE") + } + default: + if debug { + println("Adding literals ERROR:", err) + } + return err + case nil: + // Compressed litLen... + if reUsed { + if debug { + println("reused tree") + } + lh.setType(literalsBlockTreeless) + } else { + if debug { + println("new tree, size:", len(b.litEnc.OutTable)) + } + lh.setType(literalsBlockCompressed) + if debug { + _, _, err := huff0.ReadTable(out, nil) + if err != nil { + panic(err) + } + } + } + lh.setSizes(len(out), len(b.literals), single) + if debug { + printf("Compressed %d literals to %d bytes", len(b.literals), len(out)) + println("Adding literal header:", lh) + } + b.output = lh.appendTo(b.output) + b.output = append(b.output, out...) + b.litEnc.Reuse = huff0.ReusePolicyAllow + if debug { + println("Adding literals compressed") + } + } + // Sequence compression + + // Write the number of sequences + switch { + case len(b.sequences) < 128: + b.output = append(b.output, uint8(len(b.sequences))) + case len(b.sequences) < 0x7f00: // TODO: this could be wrong + n := len(b.sequences) + b.output = append(b.output, 128+uint8(n>>8), uint8(n)) + default: + n := len(b.sequences) - 0x7f00 + b.output = append(b.output, 255, uint8(n), uint8(n>>8)) + } + if debug { + println("Encoding", len(b.sequences), "sequences") + } + b.genCodes() + llEnc := b.coders.llEnc + ofEnc := b.coders.ofEnc + mlEnc := b.coders.mlEnc + err = llEnc.normalizeCount(len(b.sequences)) + if err != nil { + return err + } + err = ofEnc.normalizeCount(len(b.sequences)) + if err != nil { + return err + } + err = mlEnc.normalizeCount(len(b.sequences)) + if err != nil { + return err + } + + // Choose the best compression mode for each type. + // Will evaluate the new vs predefined and previous. + chooseComp := func(cur, prev, preDef *fseEncoder) (*fseEncoder, seqCompMode) { + // See if predefined/previous is better + hist := cur.count[:cur.symbolLen] + nSize := cur.approxSize(hist) + cur.maxHeaderSize() + predefSize := preDef.approxSize(hist) + prevSize := prev.approxSize(hist) + + // Add a small penalty for new encoders. + // Don't bother with extremely small (<2 byte gains). + nSize = nSize + (nSize+2*8*16)>>4 + switch { + case predefSize <= prevSize && predefSize <= nSize || forcePreDef: + if debug { + println("Using predefined", predefSize>>3, "<=", nSize>>3) + } + return preDef, compModePredefined + case prevSize <= nSize: + if debug { + println("Using previous", prevSize>>3, "<=", nSize>>3) + } + return prev, compModeRepeat + default: + if debug { + println("Using new, predef", predefSize>>3, ". previous:", prevSize>>3, ">", nSize>>3, "header max:", cur.maxHeaderSize()>>3, "bytes") + println("tl:", cur.actualTableLog, "symbolLen:", cur.symbolLen, "norm:", cur.norm[:cur.symbolLen], "hist", cur.count[:cur.symbolLen]) + } + return cur, compModeFSE + } + } + + // Write compression mode + var mode uint8 + if llEnc.useRLE { + mode |= uint8(compModeRLE) << 6 + llEnc.setRLE(b.sequences[0].llCode) + if debug { + println("llEnc.useRLE") + } + } else { + var m seqCompMode + llEnc, m = chooseComp(llEnc, b.coders.llPrev, &fsePredefEnc[tableLiteralLengths]) + mode |= uint8(m) << 6 + } + if ofEnc.useRLE { + mode |= uint8(compModeRLE) << 4 + ofEnc.setRLE(b.sequences[0].ofCode) + if debug { + println("ofEnc.useRLE") + } + } else { + var m seqCompMode + ofEnc, m = chooseComp(ofEnc, b.coders.ofPrev, &fsePredefEnc[tableOffsets]) + mode |= uint8(m) << 4 + } + + if mlEnc.useRLE { + mode |= uint8(compModeRLE) << 2 + mlEnc.setRLE(b.sequences[0].mlCode) + if debug { + println("mlEnc.useRLE, code: ", b.sequences[0].mlCode, "value", b.sequences[0].matchLen) + } + } else { + var m seqCompMode + mlEnc, m = chooseComp(mlEnc, b.coders.mlPrev, &fsePredefEnc[tableMatchLengths]) + mode |= uint8(m) << 2 + } + b.output = append(b.output, mode) + if debug { + printf("Compression modes: 0b%b", mode) + } + b.output, err = llEnc.writeCount(b.output) + if err != nil { + return err + } + start := len(b.output) + b.output, err = ofEnc.writeCount(b.output) + if err != nil { + return err + } + if false { + println("block:", b.output[start:], "tablelog", ofEnc.actualTableLog, "maxcount:", ofEnc.maxCount) + fmt.Printf("selected TableLog: %d, Symbol length: %d\n", ofEnc.actualTableLog, ofEnc.symbolLen) + for i, v := range ofEnc.norm[:ofEnc.symbolLen] { + fmt.Printf("%3d: %5d -> %4d \n", i, ofEnc.count[i], v) + } + } + b.output, err = mlEnc.writeCount(b.output) + if err != nil { + return err + } + + // Maybe in block? + wr := &b.wr + wr.reset(b.output) + + var ll, of, ml cState + + // Current sequence + seq := len(b.sequences) - 1 + s := b.sequences[seq] + llEnc.setBits(llBitsTable[:]) + mlEnc.setBits(mlBitsTable[:]) + ofEnc.setBits(nil) + + llTT, ofTT, mlTT := llEnc.ct.symbolTT[:256], ofEnc.ct.symbolTT[:256], mlEnc.ct.symbolTT[:256] + + // We have 3 bounds checks here (and in the loop). + // Since we are iterating backwards it is kinda hard to avoid. + llB, ofB, mlB := llTT[s.llCode], ofTT[s.ofCode], mlTT[s.mlCode] + ll.init(wr, &llEnc.ct, llB) + of.init(wr, &ofEnc.ct, ofB) + wr.flush32() + ml.init(wr, &mlEnc.ct, mlB) + + // Each of these lookups also generates a bounds check. + wr.addBits32NC(s.litLen, llB.outBits) + wr.addBits32NC(s.matchLen, mlB.outBits) + wr.flush32() + wr.addBits32NC(s.offset, ofB.outBits) + if debugSequences { + println("Encoded seq", seq, s, "codes:", s.llCode, s.mlCode, s.ofCode, "states:", ll.state, ml.state, of.state, "bits:", llB, mlB, ofB) + } + seq-- + if llEnc.maxBits+mlEnc.maxBits+ofEnc.maxBits <= 32 { + // No need to flush (common) + for seq >= 0 { + s = b.sequences[seq] + wr.flush32() + llB, ofB, mlB := llTT[s.llCode], ofTT[s.ofCode], mlTT[s.mlCode] + // tabelog max is 8 for all. + of.encode(ofB) + ml.encode(mlB) + ll.encode(llB) + wr.flush32() + + // We checked that all can stay within 32 bits + wr.addBits32NC(s.litLen, llB.outBits) + wr.addBits32NC(s.matchLen, mlB.outBits) + wr.addBits32NC(s.offset, ofB.outBits) + + if debugSequences { + println("Encoded seq", seq, s) + } + + seq-- + } + } else { + for seq >= 0 { + s = b.sequences[seq] + wr.flush32() + llB, ofB, mlB := llTT[s.llCode], ofTT[s.ofCode], mlTT[s.mlCode] + // tabelog max is below 8 for each. + of.encode(ofB) + ml.encode(mlB) + ll.encode(llB) + wr.flush32() + + // ml+ll = max 32 bits total + wr.addBits32NC(s.litLen, llB.outBits) + wr.addBits32NC(s.matchLen, mlB.outBits) + wr.flush32() + wr.addBits32NC(s.offset, ofB.outBits) + + if debugSequences { + println("Encoded seq", seq, s) + } + + seq-- + } + } + ml.flush(mlEnc.actualTableLog) + of.flush(ofEnc.actualTableLog) + ll.flush(llEnc.actualTableLog) + err = wr.close() + if err != nil { + return err + } + b.output = wr.out + + if len(b.output)-3-bhOffset >= b.size { + // Maybe even add a bigger margin. + b.litEnc.Reuse = huff0.ReusePolicyNone + return errIncompressible + } + + // Size is output minus block header. + bh.setSize(uint32(len(b.output)-bhOffset) - 3) + if debug { + println("Rewriting block header", bh) + } + _ = bh.appendTo(b.output[bhOffset:bhOffset]) + b.coders.setPrev(llEnc, mlEnc, ofEnc) + return nil +} + +var errIncompressible = errors.New("incompressible") + +func (b *blockEnc) genCodes() { + if len(b.sequences) == 0 { + // nothing to do + return + } + + if len(b.sequences) > math.MaxUint16 { + panic("can only encode up to 64K sequences") + } + // No bounds checks after here: + llH := b.coders.llEnc.Histogram()[:256] + ofH := b.coders.ofEnc.Histogram()[:256] + mlH := b.coders.mlEnc.Histogram()[:256] + for i := range llH { + llH[i] = 0 + } + for i := range ofH { + ofH[i] = 0 + } + for i := range mlH { + mlH[i] = 0 + } + + var llMax, ofMax, mlMax uint8 + for i, seq := range b.sequences { + v := llCode(seq.litLen) + seq.llCode = v + llH[v]++ + if v > llMax { + llMax = v + } + + v = ofCode(seq.offset) + seq.ofCode = v + ofH[v]++ + if v > ofMax { + ofMax = v + } + + v = mlCode(seq.matchLen) + seq.mlCode = v + mlH[v]++ + if v > mlMax { + mlMax = v + if debug && mlMax > maxMatchLengthSymbol { + panic(fmt.Errorf("mlMax > maxMatchLengthSymbol (%d), matchlen: %d", mlMax, seq.matchLen)) + } + } + b.sequences[i] = seq + } + maxCount := func(a []uint32) int { + var max uint32 + for _, v := range a { + if v > max { + max = v + } + } + return int(max) + } + if mlMax > maxMatchLengthSymbol { + panic(fmt.Errorf("mlMax > maxMatchLengthSymbol (%d)", mlMax)) + } + if ofMax > maxOffsetBits { + panic(fmt.Errorf("ofMax > maxOffsetBits (%d)", ofMax)) + } + if llMax > maxLiteralLengthSymbol { + panic(fmt.Errorf("llMax > maxLiteralLengthSymbol (%d)", llMax)) + } + + b.coders.mlEnc.HistogramFinished(mlMax, maxCount(mlH[:mlMax+1])) + b.coders.ofEnc.HistogramFinished(ofMax, maxCount(ofH[:ofMax+1])) + b.coders.llEnc.HistogramFinished(llMax, maxCount(llH[:llMax+1])) +} diff --git a/vendor/github.com/klauspost/compress/zstd/blocktype_string.go b/vendor/github.com/klauspost/compress/zstd/blocktype_string.go new file mode 100644 index 000000000..01a01e486 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/blocktype_string.go @@ -0,0 +1,85 @@ +// Code generated by "stringer -type=blockType,literalsBlockType,seqCompMode,tableIndex"; DO NOT EDIT. + +package zstd + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[blockTypeRaw-0] + _ = x[blockTypeRLE-1] + _ = x[blockTypeCompressed-2] + _ = x[blockTypeReserved-3] +} + +const _blockType_name = "blockTypeRawblockTypeRLEblockTypeCompressedblockTypeReserved" + +var _blockType_index = [...]uint8{0, 12, 24, 43, 60} + +func (i blockType) String() string { + if i >= blockType(len(_blockType_index)-1) { + return "blockType(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _blockType_name[_blockType_index[i]:_blockType_index[i+1]] +} +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[literalsBlockRaw-0] + _ = x[literalsBlockRLE-1] + _ = x[literalsBlockCompressed-2] + _ = x[literalsBlockTreeless-3] +} + +const _literalsBlockType_name = "literalsBlockRawliteralsBlockRLEliteralsBlockCompressedliteralsBlockTreeless" + +var _literalsBlockType_index = [...]uint8{0, 16, 32, 55, 76} + +func (i literalsBlockType) String() string { + if i >= literalsBlockType(len(_literalsBlockType_index)-1) { + return "literalsBlockType(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _literalsBlockType_name[_literalsBlockType_index[i]:_literalsBlockType_index[i+1]] +} +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[compModePredefined-0] + _ = x[compModeRLE-1] + _ = x[compModeFSE-2] + _ = x[compModeRepeat-3] +} + +const _seqCompMode_name = "compModePredefinedcompModeRLEcompModeFSEcompModeRepeat" + +var _seqCompMode_index = [...]uint8{0, 18, 29, 40, 54} + +func (i seqCompMode) String() string { + if i >= seqCompMode(len(_seqCompMode_index)-1) { + return "seqCompMode(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _seqCompMode_name[_seqCompMode_index[i]:_seqCompMode_index[i+1]] +} +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[tableLiteralLengths-0] + _ = x[tableOffsets-1] + _ = x[tableMatchLengths-2] +} + +const _tableIndex_name = "tableLiteralLengthstableOffsetstableMatchLengths" + +var _tableIndex_index = [...]uint8{0, 19, 31, 48} + +func (i tableIndex) String() string { + if i >= tableIndex(len(_tableIndex_index)-1) { + return "tableIndex(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _tableIndex_name[_tableIndex_index[i]:_tableIndex_index[i+1]] +} diff --git a/vendor/github.com/klauspost/compress/zstd/bytebuf.go b/vendor/github.com/klauspost/compress/zstd/bytebuf.go new file mode 100644 index 000000000..07321acb1 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/bytebuf.go @@ -0,0 +1,127 @@ +// Copyright 2019+ Klaus Post. All rights reserved. +// License information can be found in the LICENSE file. +// Based on work by Yann Collet, released under BSD License. + +package zstd + +import ( + "fmt" + "io" + "io/ioutil" +) + +type byteBuffer interface { + // Read up to 8 bytes. + // Returns nil if no more input is available. + readSmall(n int) []byte + + // Read >8 bytes. + // MAY use the destination slice. + readBig(n int, dst []byte) ([]byte, error) + + // Read a single byte. + readByte() (byte, error) + + // Skip n bytes. + skipN(n int) error +} + +// in-memory buffer +type byteBuf []byte + +func (b *byteBuf) readSmall(n int) []byte { + if debug && n > 8 { + panic(fmt.Errorf("small read > 8 (%d). use readBig", n)) + } + bb := *b + if len(bb) < n { + return nil + } + r := bb[:n] + *b = bb[n:] + return r +} + +func (b *byteBuf) readBig(n int, dst []byte) ([]byte, error) { + bb := *b + if len(bb) < n { + return nil, io.ErrUnexpectedEOF + } + r := bb[:n] + *b = bb[n:] + return r, nil +} + +func (b *byteBuf) remain() []byte { + return *b +} + +func (b *byteBuf) readByte() (byte, error) { + bb := *b + if len(bb) < 1 { + return 0, nil + } + r := bb[0] + *b = bb[1:] + return r, nil +} + +func (b *byteBuf) skipN(n int) error { + bb := *b + if len(bb) < n { + return io.ErrUnexpectedEOF + } + *b = bb[n:] + return nil +} + +// wrapper around a reader. +type readerWrapper struct { + r io.Reader + tmp [8]byte +} + +func (r *readerWrapper) readSmall(n int) []byte { + if debug && n > 8 { + panic(fmt.Errorf("small read > 8 (%d). use readBig", n)) + } + n2, err := io.ReadFull(r.r, r.tmp[:n]) + // We only really care about the actual bytes read. + if n2 != n { + if debug { + println("readSmall: got", n2, "want", n, "err", err) + } + return nil + } + return r.tmp[:n] +} + +func (r *readerWrapper) readBig(n int, dst []byte) ([]byte, error) { + if cap(dst) < n { + dst = make([]byte, n) + } + n2, err := io.ReadFull(r.r, dst[:n]) + if err == io.EOF && n > 0 { + err = io.ErrUnexpectedEOF + } + return dst[:n2], err +} + +func (r *readerWrapper) readByte() (byte, error) { + n2, err := r.r.Read(r.tmp[:1]) + if err != nil { + return 0, err + } + if n2 != 1 { + return 0, io.ErrUnexpectedEOF + } + return r.tmp[0], nil +} + +func (r *readerWrapper) skipN(n int) error { + n2, err := io.CopyN(ioutil.Discard, r.r, int64(n)) + if n2 != int64(n) { + err = io.ErrUnexpectedEOF + } + return err +} diff --git a/vendor/github.com/klauspost/compress/zstd/bytereader.go b/vendor/github.com/klauspost/compress/zstd/bytereader.go new file mode 100644 index 000000000..dc4378b64 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/bytereader.go @@ -0,0 +1,74 @@ +// Copyright 2019+ Klaus Post. All rights reserved. +// License information can be found in the LICENSE file. +// Based on work by Yann Collet, released under BSD License. + +package zstd + +// byteReader provides a byte reader that reads +// little endian values from a byte stream. +// The input stream is manually advanced. +// The reader performs no bounds checks. +type byteReader struct { + b []byte + off int +} + +// init will initialize the reader and set the input. +func (b *byteReader) init(in []byte) { + b.b = in + b.off = 0 +} + +// advance the stream b n bytes. +func (b *byteReader) advance(n uint) { + b.off += int(n) +} + +// overread returns whether we have advanced too far. +func (b *byteReader) overread() bool { + return b.off > len(b.b) +} + +// Int32 returns a little endian int32 starting at current offset. +func (b byteReader) Int32() int32 { + b2 := b.b[b.off : b.off+4 : b.off+4] + v3 := int32(b2[3]) + v2 := int32(b2[2]) + v1 := int32(b2[1]) + v0 := int32(b2[0]) + return v0 | (v1 << 8) | (v2 << 16) | (v3 << 24) +} + +// Uint8 returns the next byte +func (b *byteReader) Uint8() uint8 { + v := b.b[b.off] + return v +} + +// Uint32 returns a little endian uint32 starting at current offset. +func (b byteReader) Uint32() uint32 { + if r := b.remain(); r < 4 { + // Very rare + v := uint32(0) + for i := 1; i <= r; i++ { + v = (v << 8) | uint32(b.b[len(b.b)-i]) + } + return v + } + b2 := b.b[b.off : b.off+4 : b.off+4] + v3 := uint32(b2[3]) + v2 := uint32(b2[2]) + v1 := uint32(b2[1]) + v0 := uint32(b2[0]) + return v0 | (v1 << 8) | (v2 << 16) | (v3 << 24) +} + +// unread returns the unread portion of the input. +func (b byteReader) unread() []byte { + return b.b[b.off:] +} + +// remain will return the number of bytes remaining. +func (b byteReader) remain() int { + return len(b.b) - b.off +} diff --git a/vendor/github.com/klauspost/compress/zstd/decoder.go b/vendor/github.com/klauspost/compress/zstd/decoder.go new file mode 100644 index 000000000..35a3cda91 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/decoder.go @@ -0,0 +1,513 @@ +// Copyright 2019+ Klaus Post. All rights reserved. +// License information can be found in the LICENSE file. +// Based on work by Yann Collet, released under BSD License. + +package zstd + +import ( + "bytes" + "errors" + "io" + "sync" +) + +// Decoder provides decoding of zstandard streams. +// The decoder has been designed to operate without allocations after a warmup. +// This means that you should store the decoder for best performance. +// To re-use a stream decoder, use the Reset(r io.Reader) error to switch to another stream. +// A decoder can safely be re-used even if the previous stream failed. +// To release the resources, you must call the Close() function on a decoder. +type Decoder struct { + o decoderOptions + + // Unreferenced decoders, ready for use. + decoders chan *blockDec + + // Unreferenced decoders, ready for use. + frames chan *frameDec + + // Streams ready to be decoded. + stream chan decodeStream + + // Current read position used for Reader functionality. + current decoderState + + // Custom dictionaries + dicts map[uint32]struct{} + + // streamWg is the waitgroup for all streams + streamWg sync.WaitGroup +} + +// decoderState is used for maintaining state when the decoder +// is used for streaming. +type decoderState struct { + // current block being written to stream. + decodeOutput + + // output in order to be written to stream. + output chan decodeOutput + + // cancel remaining output. + cancel chan struct{} + + flushed bool +} + +var ( + // Check the interfaces we want to support. + _ = io.WriterTo(&Decoder{}) + _ = io.Reader(&Decoder{}) +) + +// NewReader creates a new decoder. +// A nil Reader can be provided in which case Reset can be used to start a decode. +// +// A Decoder can be used in two modes: +// +// 1) As a stream, or +// 2) For stateless decoding using DecodeAll or DecodeBuffer. +// +// Only a single stream can be decoded concurrently, but the same decoder +// can run multiple concurrent stateless decodes. It is even possible to +// use stateless decodes while a stream is being decoded. +// +// The Reset function can be used to initiate a new stream, which is will considerably +// reduce the allocations normally caused by NewReader. +func NewReader(r io.Reader, opts ...DOption) (*Decoder, error) { + initPredefined() + var d Decoder + d.o.setDefault() + for _, o := range opts { + err := o(&d.o) + if err != nil { + return nil, err + } + } + d.current.output = make(chan decodeOutput, d.o.concurrent) + d.current.flushed = true + + // Create decoders + d.decoders = make(chan *blockDec, d.o.concurrent) + d.frames = make(chan *frameDec, d.o.concurrent) + for i := 0; i < d.o.concurrent; i++ { + d.frames <- newFrameDec(d.o) + d.decoders <- newBlockDec(d.o.lowMem) + } + + if r == nil { + return &d, nil + } + return &d, d.Reset(r) +} + +// Read bytes from the decompressed stream into p. +// Returns the number of bytes written and any error that occurred. +// When the stream is done, io.EOF will be returned. +func (d *Decoder) Read(p []byte) (int, error) { + if d.stream == nil { + return 0, errors.New("no input has been initialized") + } + var n int + for { + if len(d.current.b) > 0 { + filled := copy(p, d.current.b) + p = p[filled:] + d.current.b = d.current.b[filled:] + n += filled + } + if len(p) == 0 { + break + } + if len(d.current.b) == 0 { + // We have an error and no more data + if d.current.err != nil { + break + } + if !d.nextBlock(n == 0) { + return n, nil + } + } + } + if len(d.current.b) > 0 { + if debug { + println("returning", n, "still bytes left:", len(d.current.b)) + } + // Only return error at end of block + return n, nil + } + if d.current.err != nil { + d.drainOutput() + } + if debug { + println("returning", n, d.current.err, len(d.decoders)) + } + return n, d.current.err +} + +// Reset will reset the decoder the supplied stream after the current has finished processing. +// Note that this functionality cannot be used after Close has been called. +func (d *Decoder) Reset(r io.Reader) error { + if d.current.err == ErrDecoderClosed { + return d.current.err + } + if r == nil { + return errors.New("nil Reader sent as input") + } + + if d.stream == nil { + d.stream = make(chan decodeStream, 1) + d.streamWg.Add(1) + go d.startStreamDecoder(d.stream) + } + + d.drainOutput() + + // If bytes buffer and < 1MB, do sync decoding anyway. + if bb, ok := r.(*bytes.Buffer); ok && bb.Len() < 1<<20 { + if debug { + println("*bytes.Buffer detected, doing sync decode, len:", bb.Len()) + } + b := bb.Bytes() + dst, err := d.DecodeAll(b, nil) + if err == nil { + err = io.EOF + } + d.current.b = dst + d.current.err = err + d.current.flushed = true + if debug { + println("sync decode to ", len(dst), "bytes, err:", err) + } + return nil + } + + // Remove current block. + d.current.decodeOutput = decodeOutput{} + d.current.err = nil + d.current.cancel = make(chan struct{}) + d.current.flushed = false + d.current.d = nil + + d.stream <- decodeStream{ + r: r, + output: d.current.output, + cancel: d.current.cancel, + } + return nil +} + +// drainOutput will drain the output until errEndOfStream is sent. +func (d *Decoder) drainOutput() { + if d.current.cancel != nil { + println("cancelling current") + close(d.current.cancel) + d.current.cancel = nil + } + if d.current.d != nil { + if debug { + printf("re-adding current decoder %p, decoders: %d", d.current.d, len(d.decoders)) + } + d.decoders <- d.current.d + d.current.d = nil + d.current.b = nil + } + if d.current.output == nil || d.current.flushed { + println("current already flushed") + return + } + for { + select { + case v := <-d.current.output: + if v.d != nil { + if debug { + printf("re-adding decoder %p", v.d) + } + d.decoders <- v.d + } + if v.err == errEndOfStream { + println("current flushed") + d.current.flushed = true + return + } + } + } +} + +// WriteTo writes data to w until there's no more data to write or when an error occurs. +// The return value n is the number of bytes written. +// Any error encountered during the write is also returned. +func (d *Decoder) WriteTo(w io.Writer) (int64, error) { + if d.stream == nil { + return 0, errors.New("no input has been initialized") + } + var n int64 + for { + if len(d.current.b) > 0 { + n2, err2 := w.Write(d.current.b) + n += int64(n2) + if err2 != nil && d.current.err == nil { + d.current.err = err2 + break + } + } + if d.current.err != nil { + break + } + d.nextBlock(true) + } + err := d.current.err + if err != nil { + d.drainOutput() + } + if err == io.EOF { + err = nil + } + return n, err +} + +// DecodeAll allows stateless decoding of a blob of bytes. +// Output will be appended to dst, so if the destination size is known +// you can pre-allocate the destination slice to avoid allocations. +// DecodeAll can be used concurrently. +// The Decoder concurrency limits will be respected. +func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { + if d.current.err == ErrDecoderClosed { + return dst, ErrDecoderClosed + } + + // Grab a block decoder and frame decoder. + block, frame := <-d.decoders, <-d.frames + defer func() { + if debug { + printf("re-adding decoder: %p", block) + } + d.decoders <- block + frame.rawInput = nil + frame.bBuf = nil + d.frames <- frame + }() + frame.bBuf = input + + for { + err := frame.reset(&frame.bBuf) + if err == io.EOF { + return dst, nil + } + if err != nil { + return dst, err + } + if frame.FrameContentSize > d.o.maxDecodedSize-uint64(len(dst)) { + return dst, ErrDecoderSizeExceeded + } + if frame.FrameContentSize > 0 && frame.FrameContentSize < 1<<30 { + // Never preallocate moe than 1 GB up front. + if uint64(cap(dst)) < frame.FrameContentSize { + dst2 := make([]byte, len(dst), len(dst)+int(frame.FrameContentSize)) + copy(dst2, dst) + dst = dst2 + } + } + if cap(dst) == 0 { + // Allocate window size * 2 by default if nothing is provided and we didn't get frame content size. + size := frame.WindowSize * 2 + // Cap to 1 MB. + if size > 1<<20 { + size = 1 << 20 + } + dst = make([]byte, 0, frame.WindowSize) + } + + dst, err = frame.runDecoder(dst, block) + if err != nil { + return dst, err + } + if len(frame.bBuf) == 0 { + break + } + } + return dst, nil +} + +// nextBlock returns the next block. +// If an error occurs d.err will be set. +// Optionally the function can block for new output. +// If non-blocking mode is used the returned boolean will be false +// if no data was available without blocking. +func (d *Decoder) nextBlock(blocking bool) (ok bool) { + if d.current.d != nil { + if debug { + printf("re-adding current decoder %p", d.current.d) + } + d.decoders <- d.current.d + d.current.d = nil + } + if d.current.err != nil { + // Keep error state. + return blocking + } + + if blocking { + d.current.decodeOutput = <-d.current.output + } else { + select { + case d.current.decodeOutput = <-d.current.output: + default: + return false + } + } + if debug { + println("got", len(d.current.b), "bytes, error:", d.current.err) + } + return true +} + +// Close will release all resources. +// It is NOT possible to reuse the decoder after this. +func (d *Decoder) Close() { + if d.current.err == ErrDecoderClosed { + return + } + d.drainOutput() + if d.stream != nil { + close(d.stream) + d.streamWg.Wait() + d.stream = nil + } + if d.decoders != nil { + close(d.decoders) + for dec := range d.decoders { + dec.Close() + } + d.decoders = nil + } + if d.current.d != nil { + d.current.d.Close() + d.current.d = nil + } + d.current.err = ErrDecoderClosed +} + +// IOReadCloser returns the decoder as an io.ReadCloser for convenience. +// Any changes to the decoder will be reflected, so the returned ReadCloser +// can be reused along with the decoder. +// io.WriterTo is also supported by the returned ReadCloser. +func (d *Decoder) IOReadCloser() io.ReadCloser { + return closeWrapper{d: d} +} + +// closeWrapper wraps a function call as a closer. +type closeWrapper struct { + d *Decoder +} + +// WriteTo forwards WriteTo calls to the decoder. +func (c closeWrapper) WriteTo(w io.Writer) (n int64, err error) { + return c.d.WriteTo(w) +} + +// Read forwards read calls to the decoder. +func (c closeWrapper) Read(p []byte) (n int, err error) { + return c.d.Read(p) +} + +// Close closes the decoder. +func (c closeWrapper) Close() error { + c.d.Close() + return nil +} + +type decodeOutput struct { + d *blockDec + b []byte + err error +} + +type decodeStream struct { + r io.Reader + + // Blocks ready to be written to output. + output chan decodeOutput + + // cancel reading from the input + cancel chan struct{} +} + +// errEndOfStream indicates that everything from the stream was read. +var errEndOfStream = errors.New("end-of-stream") + +// Create Decoder: +// Spawn n block decoders. These accept tasks to decode a block. +// Create goroutine that handles stream processing, this will send history to decoders as they are available. +// Decoders update the history as they decode. +// When a block is returned: +// a) history is sent to the next decoder, +// b) content written to CRC. +// c) return data to WRITER. +// d) wait for next block to return data. +// Once WRITTEN, the decoders reused by the writer frame decoder for re-use. +func (d *Decoder) startStreamDecoder(inStream chan decodeStream) { + defer d.streamWg.Done() + frame := newFrameDec(d.o) + for stream := range inStream { + if debug { + println("got new stream") + } + br := readerWrapper{r: stream.r} + decodeStream: + for { + err := frame.reset(&br) + if debug && err != nil { + println("Frame decoder returned", err) + } + if err != nil { + stream.output <- decodeOutput{ + err: err, + } + break + } + if debug { + println("starting frame decoder") + } + + // This goroutine will forward history between frames. + frame.frameDone.Add(1) + frame.initAsync() + + go frame.startDecoder(stream.output) + decodeFrame: + // Go through all blocks of the frame. + for { + dec := <-d.decoders + select { + case <-stream.cancel: + if !frame.sendErr(dec, io.EOF) { + // To not let the decoder dangle, send it back. + stream.output <- decodeOutput{d: dec} + } + break decodeStream + default: + } + err := frame.next(dec) + switch err { + case io.EOF: + // End of current frame, no error + println("EOF on next block") + break decodeFrame + case nil: + continue + default: + println("block decoder returned", err) + break decodeStream + } + } + // All blocks have started decoding, check if there are more frames. + println("waiting for done") + frame.frameDone.Wait() + println("done waiting...") + } + frame.frameDone.Wait() + println("Sending EOS") + stream.output <- decodeOutput{err: errEndOfStream} + } +} diff --git a/vendor/github.com/klauspost/compress/zstd/decoder_options.go b/vendor/github.com/klauspost/compress/zstd/decoder_options.go new file mode 100644 index 000000000..2ac9cd2dd --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/decoder_options.go @@ -0,0 +1,68 @@ +// Copyright 2019+ Klaus Post. All rights reserved. +// License information can be found in the LICENSE file. +// Based on work by Yann Collet, released under BSD License. + +package zstd + +import ( + "errors" + "fmt" + "runtime" +) + +// DOption is an option for creating a decoder. +type DOption func(*decoderOptions) error + +// options retains accumulated state of multiple options. +type decoderOptions struct { + lowMem bool + concurrent int + maxDecodedSize uint64 +} + +func (o *decoderOptions) setDefault() { + *o = decoderOptions{ + // use less ram: true for now, but may change. + lowMem: true, + concurrent: runtime.GOMAXPROCS(0), + } + o.maxDecodedSize = 1 << 63 +} + +// WithDecoderLowmem will set whether to use a lower amount of memory, +// but possibly have to allocate more while running. +func WithDecoderLowmem(b bool) DOption { + return func(o *decoderOptions) error { o.lowMem = b; return nil } +} + +// WithDecoderConcurrency will set the concurrency, +// meaning the maximum number of decoders to run concurrently. +// The value supplied must be at least 1. +// By default this will be set to GOMAXPROCS. +func WithDecoderConcurrency(n int) DOption { + return func(o *decoderOptions) error { + if n <= 0 { + return fmt.Errorf("Concurrency must be at least 1") + } + o.concurrent = n + return nil + } +} + +// WithDecoderMaxMemory allows to set a maximum decoded size for in-memory +// non-streaming operations or maximum window size for streaming operations. +// This can be used to control memory usage of potentially hostile content. +// For streaming operations, the maximum window size is capped at 1<<30 bytes. +// Maximum and default is 1 << 63 bytes. +func WithDecoderMaxMemory(n uint64) DOption { + return func(o *decoderOptions) error { + if n == 0 { + return errors.New("WithDecoderMaxMemory must be at least 1") + } + if n > 1<<63 { + return fmt.Errorf("WithDecoderMaxmemory must be less than 1 << 63") + } + o.maxDecodedSize = n + return nil + } +} diff --git a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go new file mode 100644 index 000000000..ee3b09b02 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go @@ -0,0 +1,726 @@ +// Copyright 2019+ Klaus Post. All rights reserved. +// License information can be found in the LICENSE file. +// Based on work by Yann Collet, released under BSD License. + +package zstd + +const ( + dFastLongTableBits = 17 // Bits used in the long match table + dFastLongTableSize = 1 << dFastLongTableBits // Size of the table + dFastLongTableMask = dFastLongTableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks. + + dFastShortTableBits = tableBits // Bits used in the short match table + dFastShortTableSize = 1 << dFastShortTableBits // Size of the table + dFastShortTableMask = dFastShortTableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks. +) + +type doubleFastEncoder struct { + fastEncoder + longTable [dFastLongTableSize]tableEntry +} + +// Encode mimmics functionality in zstd_dfast.c +func (e *doubleFastEncoder) Encode(blk *blockEnc, src []byte) { + const ( + // Input margin is the number of bytes we read (8) + // and the maximum we will read ahead (2) + inputMargin = 8 + 2 + minNonLiteralBlockSize = 16 + ) + + // Protect against e.cur wraparound. + for e.cur > (1<<30)+e.maxMatchOff { + if len(e.hist) == 0 { + for i := range e.table[:] { + e.table[i] = tableEntry{} + } + for i := range e.longTable[:] { + e.longTable[i] = tableEntry{} + } + e.cur = e.maxMatchOff + break + } + // Shift down everything in the table that isn't already too far away. + minOff := e.cur + int32(len(e.hist)) - e.maxMatchOff + for i := range e.table[:] { + v := e.table[i].offset + if v < minOff { + v = 0 + } else { + v = v - e.cur + e.maxMatchOff + } + e.table[i].offset = v + } + for i := range e.longTable[:] { + v := e.longTable[i].offset + if v < minOff { + v = 0 + } else { + v = v - e.cur + e.maxMatchOff + } + e.longTable[i].offset = v + } + e.cur = e.maxMatchOff + } + + s := e.addBlock(src) + blk.size = len(src) + if len(src) < minNonLiteralBlockSize { + blk.extraLits = len(src) + blk.literals = blk.literals[:len(src)] + copy(blk.literals, src) + return + } + + // Override src + src = e.hist + sLimit := int32(len(src)) - inputMargin + // stepSize is the number of bytes to skip on every main loop iteration. + // It should be >= 1. + stepSize := int32(e.o.targetLength) + if stepSize == 0 { + stepSize++ + } + + const kSearchStrength = 8 + + // nextEmit is where in src the next emitLiteral should start from. + nextEmit := s + cv := load6432(src, s) + + // Relative offsets + offset1 := int32(blk.recentOffsets[0]) + offset2 := int32(blk.recentOffsets[1]) + + addLiterals := func(s *seq, until int32) { + if until == nextEmit { + return + } + blk.literals = append(blk.literals, src[nextEmit:until]...) + s.litLen = uint32(until - nextEmit) + } + if debug { + println("recent offsets:", blk.recentOffsets) + } + +encodeLoop: + for { + var t int32 + // We allow the encoder to optionally turn off repeat offsets across blocks + canRepeat := len(blk.sequences) > 2 + + for { + if debug && canRepeat && offset1 == 0 { + panic("offset0 was 0") + } + + nextHashS := hash5(cv, dFastShortTableBits) + nextHashL := hash8(cv, dFastLongTableBits) + candidateL := e.longTable[nextHashL] + candidateS := e.table[nextHashS] + + const repOff = 1 + repIndex := s - offset1 + repOff + entry := tableEntry{offset: s + e.cur, val: uint32(cv)} + e.longTable[nextHashL] = entry + e.table[nextHashS] = entry + + if canRepeat { + if repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>(repOff*8)) { + // Consider history as well. + var seq seq + lenght := 4 + e.matchlen(s+4+repOff, repIndex+4, src) + + seq.matchLen = uint32(lenght - zstdMinMatch) + + // We might be able to match backwards. + // Extend as long as we can. + start := s + repOff + // We end the search early, so we don't risk 0 literals + // and have to do special offset treatment. + startLimit := nextEmit + 1 + + tMin := s - e.maxMatchOff + if tMin < 0 { + tMin = 0 + } + for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { + repIndex-- + start-- + seq.matchLen++ + } + addLiterals(&seq, start) + + // rep 0 + seq.offset = 1 + if debugSequences { + println("repeat sequence", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + s += lenght + repOff + nextEmit = s + if s >= sLimit { + if debug { + println("repeat ended", s, lenght) + + } + break encodeLoop + } + cv = load6432(src, s) + continue + } + const repOff2 = 1 + // We deviate from the reference encoder and also check offset 2. + // Slower and not consistently better, so disabled. + // repIndex = s - offset2 + repOff2 + if false && repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>(repOff2*8)) { + // Consider history as well. + var seq seq + lenght := 4 + e.matchlen(s+4+repOff2, repIndex+4, src) + + seq.matchLen = uint32(lenght - zstdMinMatch) + + // We might be able to match backwards. + // Extend as long as we can. + start := s + repOff2 + // We end the search early, so we don't risk 0 literals + // and have to do special offset treatment. + startLimit := nextEmit + 1 + + tMin := s - e.maxMatchOff + if tMin < 0 { + tMin = 0 + } + for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { + repIndex-- + start-- + seq.matchLen++ + } + addLiterals(&seq, start) + + // rep 2 + seq.offset = 2 + if debugSequences { + println("repeat sequence 2", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + s += lenght + repOff2 + nextEmit = s + if s >= sLimit { + if debug { + println("repeat ended", s, lenght) + + } + break encodeLoop + } + cv = load6432(src, s) + // Swap offsets + offset1, offset2 = offset2, offset1 + continue + } + } + // Find the offsets of our two matches. + coffsetL := s - (candidateL.offset - e.cur) + coffsetS := s - (candidateS.offset - e.cur) + + // Check if we have a long match. + if coffsetL < e.maxMatchOff && uint32(cv) == candidateL.val { + // Found a long match, likely at least 8 bytes. + // Reference encoder checks all 8 bytes, we only check 4, + // but the likelihood of both the first 4 bytes and the hash matching should be enough. + t = candidateL.offset - e.cur + if debug && s <= t { + panic("s <= t") + } + if debug && s-t > e.maxMatchOff { + panic("s - t >e.maxMatchOff") + } + if debugMatches { + println("long match") + } + break + } + + // Check if we have a short match. + if coffsetS < e.maxMatchOff && uint32(cv) == candidateS.val { + // found a regular match + // See if we can find a long match at s+1 + const checkAt = 1 + cv := load6432(src, s+checkAt) + nextHashL = hash8(cv, dFastLongTableBits) + candidateL = e.longTable[nextHashL] + coffsetL = s - (candidateL.offset - e.cur) + checkAt + + // We can store it, since we have at least a 4 byte match. + e.longTable[nextHashL] = tableEntry{offset: s + checkAt + e.cur, val: uint32(cv)} + if coffsetL < e.maxMatchOff && uint32(cv) == candidateL.val { + // Found a long match, likely at least 8 bytes. + // Reference encoder checks all 8 bytes, we only check 4, + // but the likelihood of both the first 4 bytes and the hash matching should be enough. + t = candidateL.offset - e.cur + s += checkAt + if debugMatches { + println("long match (after short)") + } + break + } + + t = candidateS.offset - e.cur + if debug && s <= t { + panic("s <= t") + } + if debug && s-t > e.maxMatchOff { + panic("s - t >e.maxMatchOff") + } + if debug && t < 0 { + panic("t<0") + } + if debugMatches { + println("short match") + } + break + } + + // No match found, move forward in input. + s += stepSize + ((s - nextEmit) >> (kSearchStrength - 1)) + if s >= sLimit { + break encodeLoop + } + cv = load6432(src, s) + } + + // A 4-byte match has been found. Update recent offsets. + // We'll later see if more than 4 bytes. + offset2 = offset1 + offset1 = s - t + + if debug && s <= t { + panic("s <= t") + } + + if debug && canRepeat && int(offset1) > len(src) { + panic("invalid offset") + } + + // Extend the 4-byte match as long as possible. + l := e.matchlen(s+4, t+4, src) + 4 + + // Extend backwards + tMin := s - e.maxMatchOff + if tMin < 0 { + tMin = 0 + } + for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength { + s-- + t-- + l++ + } + + // Write our sequence + var seq seq + seq.litLen = uint32(s - nextEmit) + seq.matchLen = uint32(l - zstdMinMatch) + if seq.litLen > 0 { + blk.literals = append(blk.literals, src[nextEmit:s]...) + } + seq.offset = uint32(s-t) + 3 + s += l + if debugSequences { + println("sequence", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + nextEmit = s + if s >= sLimit { + break encodeLoop + } + + // Index match start+1 (long) and start+2 (short) + index0 := s - l + 1 + // Index match end-2 (long) and end-1 (short) + index1 := s - 2 + + cv0 := load6432(src, index0) + cv1 := load6432(src, index1) + te0 := tableEntry{offset: index0 + e.cur, val: uint32(cv0)} + te1 := tableEntry{offset: index1 + e.cur, val: uint32(cv1)} + e.longTable[hash8(cv0, dFastLongTableBits)] = te0 + e.longTable[hash8(cv1, dFastLongTableBits)] = te1 + cv0 >>= 8 + cv1 >>= 8 + te0.offset++ + te1.offset++ + te0.val = uint32(cv0) + te1.val = uint32(cv1) + e.table[hash5(cv0, dFastShortTableBits)] = te0 + e.table[hash5(cv1, dFastShortTableBits)] = te1 + + cv = load6432(src, s) + + if !canRepeat { + continue + } + + // Check offset 2 + for { + o2 := s - offset2 + if load3232(src, o2) != uint32(cv) { + // Do regular search + break + } + + // Store this, since we have it. + nextHashS := hash5(cv1>>8, dFastShortTableBits) + nextHashL := hash8(cv, dFastLongTableBits) + + // We have at least 4 byte match. + // No need to check backwards. We come straight from a match + l := 4 + e.matchlen(s+4, o2+4, src) + + entry := tableEntry{offset: s + e.cur, val: uint32(cv)} + e.longTable[nextHashL] = entry + e.table[nextHashS] = entry + seq.matchLen = uint32(l) - zstdMinMatch + seq.litLen = 0 + + // Since litlen is always 0, this is offset 1. + seq.offset = 1 + s += l + nextEmit = s + if debugSequences { + println("sequence", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + + // Swap offset 1 and 2. + offset1, offset2 = offset2, offset1 + if s >= sLimit { + // Finished + break encodeLoop + } + cv = load6432(src, s) + } + } + + if int(nextEmit) < len(src) { + blk.literals = append(blk.literals, src[nextEmit:]...) + blk.extraLits = len(src) - int(nextEmit) + } + blk.recentOffsets[0] = uint32(offset1) + blk.recentOffsets[1] = uint32(offset2) + if debug { + println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) + } +} + +// EncodeNoHist will encode a block with no history and no following blocks. +// Most notable difference is that src will not be copied for history and +// we do not need to check for max match length. +func (e *doubleFastEncoder) EncodeNoHist(blk *blockEnc, src []byte) { + const ( + // Input margin is the number of bytes we read (8) + // and the maximum we will read ahead (2) + inputMargin = 8 + 2 + minNonLiteralBlockSize = 16 + ) + + // Protect against e.cur wraparound. + if e.cur > (1<<30)+e.maxMatchOff { + for i := range e.table[:] { + e.table[i] = tableEntry{} + } + for i := range e.longTable[:] { + e.longTable[i] = tableEntry{} + } + e.cur = e.maxMatchOff + } + + s := int32(0) + blk.size = len(src) + if len(src) < minNonLiteralBlockSize { + blk.extraLits = len(src) + blk.literals = blk.literals[:len(src)] + copy(blk.literals, src) + return + } + + // Override src + sLimit := int32(len(src)) - inputMargin + // stepSize is the number of bytes to skip on every main loop iteration. + // It should be >= 1. + stepSize := int32(e.o.targetLength) + if stepSize == 0 { + stepSize++ + } + + const kSearchStrength = 8 + + // nextEmit is where in src the next emitLiteral should start from. + nextEmit := s + cv := load6432(src, s) + + // Relative offsets + offset1 := int32(blk.recentOffsets[0]) + offset2 := int32(blk.recentOffsets[1]) + + addLiterals := func(s *seq, until int32) { + if until == nextEmit { + return + } + blk.literals = append(blk.literals, src[nextEmit:until]...) + s.litLen = uint32(until - nextEmit) + } + if debug { + println("recent offsets:", blk.recentOffsets) + } + +encodeLoop: + for { + var t int32 + for { + + nextHashS := hash5(cv, dFastShortTableBits) + nextHashL := hash8(cv, dFastLongTableBits) + candidateL := e.longTable[nextHashL] + candidateS := e.table[nextHashS] + + const repOff = 1 + repIndex := s - offset1 + repOff + entry := tableEntry{offset: s + e.cur, val: uint32(cv)} + e.longTable[nextHashL] = entry + e.table[nextHashS] = entry + + if len(blk.sequences) > 2 { + if load3232(src, repIndex) == uint32(cv>>(repOff*8)) { + // Consider history as well. + var seq seq + //length := 4 + e.matchlen(s+4+repOff, repIndex+4, src) + length := 4 + int32(matchLen(src[s+4+repOff:], src[repIndex+4:])) + + seq.matchLen = uint32(length - zstdMinMatch) + + // We might be able to match backwards. + // Extend as long as we can. + start := s + repOff + // We end the search early, so we don't risk 0 literals + // and have to do special offset treatment. + startLimit := nextEmit + 1 + + tMin := s - e.maxMatchOff + if tMin < 0 { + tMin = 0 + } + for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] { + repIndex-- + start-- + seq.matchLen++ + } + addLiterals(&seq, start) + + // rep 0 + seq.offset = 1 + if debugSequences { + println("repeat sequence", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + s += length + repOff + nextEmit = s + if s >= sLimit { + if debug { + println("repeat ended", s, length) + + } + break encodeLoop + } + cv = load6432(src, s) + continue + } + } + // Find the offsets of our two matches. + coffsetL := s - (candidateL.offset - e.cur) + coffsetS := s - (candidateS.offset - e.cur) + + // Check if we have a long match. + if coffsetL < e.maxMatchOff && uint32(cv) == candidateL.val { + // Found a long match, likely at least 8 bytes. + // Reference encoder checks all 8 bytes, we only check 4, + // but the likelihood of both the first 4 bytes and the hash matching should be enough. + t = candidateL.offset - e.cur + if debug && s <= t { + panic("s <= t") + } + if debug && s-t > e.maxMatchOff { + panic("s - t >e.maxMatchOff") + } + if debugMatches { + println("long match") + } + break + } + + // Check if we have a short match. + if coffsetS < e.maxMatchOff && uint32(cv) == candidateS.val { + // found a regular match + // See if we can find a long match at s+1 + const checkAt = 1 + cv := load6432(src, s+checkAt) + nextHashL = hash8(cv, dFastLongTableBits) + candidateL = e.longTable[nextHashL] + coffsetL = s - (candidateL.offset - e.cur) + checkAt + + // We can store it, since we have at least a 4 byte match. + e.longTable[nextHashL] = tableEntry{offset: s + checkAt + e.cur, val: uint32(cv)} + if coffsetL < e.maxMatchOff && uint32(cv) == candidateL.val { + // Found a long match, likely at least 8 bytes. + // Reference encoder checks all 8 bytes, we only check 4, + // but the likelihood of both the first 4 bytes and the hash matching should be enough. + t = candidateL.offset - e.cur + s += checkAt + if debugMatches { + println("long match (after short)") + } + break + } + + t = candidateS.offset - e.cur + if debug && s <= t { + panic("s <= t") + } + if debug && s-t > e.maxMatchOff { + panic("s - t >e.maxMatchOff") + } + if debug && t < 0 { + panic("t<0") + } + if debugMatches { + println("short match") + } + break + } + + // No match found, move forward in input. + s += stepSize + ((s - nextEmit) >> (kSearchStrength - 1)) + if s >= sLimit { + break encodeLoop + } + cv = load6432(src, s) + } + + // A 4-byte match has been found. Update recent offsets. + // We'll later see if more than 4 bytes. + offset2 = offset1 + offset1 = s - t + + if debug && s <= t { + panic("s <= t") + } + + // Extend the 4-byte match as long as possible. + //l := e.matchlen(s+4, t+4, src) + 4 + l := int32(matchLen(src[s+4:], src[t+4:])) + 4 + + // Extend backwards + tMin := s - e.maxMatchOff + if tMin < 0 { + tMin = 0 + } + for t > tMin && s > nextEmit && src[t-1] == src[s-1] { + s-- + t-- + l++ + } + + // Write our sequence + var seq seq + seq.litLen = uint32(s - nextEmit) + seq.matchLen = uint32(l - zstdMinMatch) + if seq.litLen > 0 { + blk.literals = append(blk.literals, src[nextEmit:s]...) + } + seq.offset = uint32(s-t) + 3 + s += l + if debugSequences { + println("sequence", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + nextEmit = s + if s >= sLimit { + break encodeLoop + } + + // Index match start+1 (long) and start+2 (short) + index0 := s - l + 1 + // Index match end-2 (long) and end-1 (short) + index1 := s - 2 + + cv0 := load6432(src, index0) + cv1 := load6432(src, index1) + te0 := tableEntry{offset: index0 + e.cur, val: uint32(cv0)} + te1 := tableEntry{offset: index1 + e.cur, val: uint32(cv1)} + e.longTable[hash8(cv0, dFastLongTableBits)] = te0 + e.longTable[hash8(cv1, dFastLongTableBits)] = te1 + cv0 >>= 8 + cv1 >>= 8 + te0.offset++ + te1.offset++ + te0.val = uint32(cv0) + te1.val = uint32(cv1) + e.table[hash5(cv0, dFastShortTableBits)] = te0 + e.table[hash5(cv1, dFastShortTableBits)] = te1 + + cv = load6432(src, s) + + if len(blk.sequences) <= 2 { + continue + } + + // Check offset 2 + for { + o2 := s - offset2 + if load3232(src, o2) != uint32(cv) { + // Do regular search + break + } + + // Store this, since we have it. + nextHashS := hash5(cv1>>8, dFastShortTableBits) + nextHashL := hash8(cv, dFastLongTableBits) + + // We have at least 4 byte match. + // No need to check backwards. We come straight from a match + //l := 4 + e.matchlen(s+4, o2+4, src) + l := 4 + int32(matchLen(src[s+4:], src[o2+4:])) + + entry := tableEntry{offset: s + e.cur, val: uint32(cv)} + e.longTable[nextHashL] = entry + e.table[nextHashS] = entry + seq.matchLen = uint32(l) - zstdMinMatch + seq.litLen = 0 + + // Since litlen is always 0, this is offset 1. + seq.offset = 1 + s += l + nextEmit = s + if debugSequences { + println("sequence", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + + // Swap offset 1 and 2. + offset1, offset2 = offset2, offset1 + if s >= sLimit { + // Finished + break encodeLoop + } + cv = load6432(src, s) + } + } + + if int(nextEmit) < len(src) { + blk.literals = append(blk.literals, src[nextEmit:]...) + blk.extraLits = len(src) - int(nextEmit) + } + if debug { + println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) + } + +} diff --git a/vendor/github.com/klauspost/compress/zstd/enc_fast.go b/vendor/github.com/klauspost/compress/zstd/enc_fast.go new file mode 100644 index 000000000..0bdddac5b --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/enc_fast.go @@ -0,0 +1,656 @@ +// Copyright 2019+ Klaus Post. All rights reserved. +// License information can be found in the LICENSE file. +// Based on work by Yann Collet, released under BSD License. + +package zstd + +import ( + "math/bits" + + "github.com/klauspost/compress/zstd/internal/xxhash" +) + +const ( + tableBits = 15 // Bits used in the table + tableSize = 1 << tableBits // Size of the table + tableMask = tableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks. + maxMatchLength = 131074 +) + +type tableEntry struct { + val uint32 + offset int32 +} + +type fastEncoder struct { + o encParams + // cur is the offset at the start of hist + cur int32 + // maximum offset. Should be at least 2x block size. + maxMatchOff int32 + hist []byte + crc *xxhash.Digest + table [tableSize]tableEntry + tmp [8]byte + blk *blockEnc +} + +// CRC returns the underlying CRC writer. +func (e *fastEncoder) CRC() *xxhash.Digest { + return e.crc +} + +// AppendCRC will append the CRC to the destination slice and return it. +func (e *fastEncoder) AppendCRC(dst []byte) []byte { + crc := e.crc.Sum(e.tmp[:0]) + dst = append(dst, crc[7], crc[6], crc[5], crc[4]) + return dst +} + +// WindowSize returns the window size of the encoder, +// or a window size small enough to contain the input size, if > 0. +func (e *fastEncoder) WindowSize(size int) int32 { + if size > 0 && size < int(e.maxMatchOff) { + b := int32(1) << uint(bits.Len(uint(size))) + // Keep minimum window. + if b < 1024 { + b = 1024 + } + return b + } + return e.maxMatchOff +} + +// Block returns the current block. +func (e *fastEncoder) Block() *blockEnc { + return e.blk +} + +// Encode mimmics functionality in zstd_fast.c +func (e *fastEncoder) Encode(blk *blockEnc, src []byte) { + const ( + inputMargin = 8 + minNonLiteralBlockSize = 1 + 1 + inputMargin + ) + + // Protect against e.cur wraparound. + for e.cur > (1<<30)+e.maxMatchOff { + if len(e.hist) == 0 { + for i := range e.table[:] { + e.table[i] = tableEntry{} + } + e.cur = e.maxMatchOff + break + } + // Shift down everything in the table that isn't already too far away. + minOff := e.cur + int32(len(e.hist)) - e.maxMatchOff + for i := range e.table[:] { + v := e.table[i].offset + if v < minOff { + v = 0 + } else { + v = v - e.cur + e.maxMatchOff + } + e.table[i].offset = v + } + e.cur = e.maxMatchOff + } + + s := e.addBlock(src) + blk.size = len(src) + if len(src) < minNonLiteralBlockSize { + blk.extraLits = len(src) + blk.literals = blk.literals[:len(src)] + copy(blk.literals, src) + return + } + + // Override src + src = e.hist + sLimit := int32(len(src)) - inputMargin + // stepSize is the number of bytes to skip on every main loop iteration. + // It should be >= 2. + stepSize := int32(e.o.targetLength) + if stepSize == 0 { + stepSize++ + } + stepSize++ + + // TEMPLATE + const hashLog = tableBits + // seems global, but would be nice to tweak. + const kSearchStrength = 8 + + // nextEmit is where in src the next emitLiteral should start from. + nextEmit := s + cv := load6432(src, s) + + // Relative offsets + offset1 := int32(blk.recentOffsets[0]) + offset2 := int32(blk.recentOffsets[1]) + + addLiterals := func(s *seq, until int32) { + if until == nextEmit { + return + } + blk.literals = append(blk.literals, src[nextEmit:until]...) + s.litLen = uint32(until - nextEmit) + } + if debug { + println("recent offsets:", blk.recentOffsets) + } + +encodeLoop: + for { + // t will contain the match offset when we find one. + // When existing the search loop, we have already checked 4 bytes. + var t int32 + + // We will not use repeat offsets across blocks. + // By not using them for the first 3 matches + canRepeat := len(blk.sequences) > 2 + + for { + if debug && canRepeat && offset1 == 0 { + panic("offset0 was 0") + } + + nextHash := hash6(cv, hashLog) + nextHash2 := hash6(cv>>8, hashLog) + candidate := e.table[nextHash] + candidate2 := e.table[nextHash2] + repIndex := s - offset1 + 2 + + e.table[nextHash] = tableEntry{offset: s + e.cur, val: uint32(cv)} + e.table[nextHash2] = tableEntry{offset: s + e.cur + 1, val: uint32(cv >> 8)} + + if canRepeat && repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>16) { + // Consider history as well. + var seq seq + lenght := 4 + e.matchlen(s+6, repIndex+4, src) + + seq.matchLen = uint32(lenght - zstdMinMatch) + + // We might be able to match backwards. + // Extend as long as we can. + start := s + 2 + // We end the search early, so we don't risk 0 literals + // and have to do special offset treatment. + startLimit := nextEmit + 1 + + sMin := s - e.maxMatchOff + if sMin < 0 { + sMin = 0 + } + for repIndex > sMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch { + repIndex-- + start-- + seq.matchLen++ + } + addLiterals(&seq, start) + + // rep 0 + seq.offset = 1 + if debugSequences { + println("repeat sequence", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + s += lenght + 2 + nextEmit = s + if s >= sLimit { + if debug { + println("repeat ended", s, lenght) + + } + break encodeLoop + } + cv = load6432(src, s) + continue + } + coffset0 := s - (candidate.offset - e.cur) + coffset1 := s - (candidate2.offset - e.cur) + 1 + if coffset0 < e.maxMatchOff && uint32(cv) == candidate.val { + // found a regular match + t = candidate.offset - e.cur + if debug && s <= t { + panic("s <= t") + } + if debug && s-t > e.maxMatchOff { + panic("s - t >e.maxMatchOff") + } + break + } + + if coffset1 < e.maxMatchOff && uint32(cv>>8) == candidate2.val { + // found a regular match + t = candidate2.offset - e.cur + s++ + if debug && s <= t { + panic("s <= t") + } + if debug && s-t > e.maxMatchOff { + panic("s - t >e.maxMatchOff") + } + if debug && t < 0 { + panic("t<0") + } + break + } + s += stepSize + ((s - nextEmit) >> (kSearchStrength - 1)) + if s >= sLimit { + break encodeLoop + } + cv = load6432(src, s) + } + // A 4-byte match has been found. We'll later see if more than 4 bytes. + offset2 = offset1 + offset1 = s - t + + if debug && s <= t { + panic("s <= t") + } + + if debug && canRepeat && int(offset1) > len(src) { + panic("invalid offset") + } + + // Extend the 4-byte match as long as possible. + l := e.matchlen(s+4, t+4, src) + 4 + + // Extend backwards + tMin := s - e.maxMatchOff + if tMin < 0 { + tMin = 0 + } + for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength { + s-- + t-- + l++ + } + + // Write our sequence. + var seq seq + seq.litLen = uint32(s - nextEmit) + seq.matchLen = uint32(l - zstdMinMatch) + if seq.litLen > 0 { + blk.literals = append(blk.literals, src[nextEmit:s]...) + } + // Don't use repeat offsets + seq.offset = uint32(s-t) + 3 + s += l + if debugSequences { + println("sequence", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + nextEmit = s + if s >= sLimit { + break encodeLoop + } + cv = load6432(src, s) + + // Check offset 2 + if o2 := s - offset2; canRepeat && load3232(src, o2) == uint32(cv) { + // We have at least 4 byte match. + // No need to check backwards. We come straight from a match + l := 4 + e.matchlen(s+4, o2+4, src) + + // Store this, since we have it. + nextHash := hash6(cv, hashLog) + e.table[nextHash] = tableEntry{offset: s + e.cur, val: uint32(cv)} + seq.matchLen = uint32(l) - zstdMinMatch + seq.litLen = 0 + // Since litlen is always 0, this is offset 1. + seq.offset = 1 + s += l + nextEmit = s + if debugSequences { + println("sequence", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + + // Swap offset 1 and 2. + offset1, offset2 = offset2, offset1 + if s >= sLimit { + break encodeLoop + } + // Prepare next loop. + cv = load6432(src, s) + } + } + + if int(nextEmit) < len(src) { + blk.literals = append(blk.literals, src[nextEmit:]...) + blk.extraLits = len(src) - int(nextEmit) + } + blk.recentOffsets[0] = uint32(offset1) + blk.recentOffsets[1] = uint32(offset2) + if debug { + println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) + } +} + +// EncodeNoHist will encode a block with no history and no following blocks. +// Most notable difference is that src will not be copied for history and +// we do not need to check for max match length. +func (e *fastEncoder) EncodeNoHist(blk *blockEnc, src []byte) { + const ( + inputMargin = 8 + minNonLiteralBlockSize = 1 + 1 + inputMargin + ) + if debug { + if len(src) > maxBlockSize { + panic("src too big") + } + } + // Protect against e.cur wraparound. + if e.cur > (1<<30)+e.maxMatchOff { + for i := range e.table[:] { + e.table[i] = tableEntry{} + } + e.cur = e.maxMatchOff + } + + s := int32(0) + blk.size = len(src) + if len(src) < minNonLiteralBlockSize { + blk.extraLits = len(src) + blk.literals = blk.literals[:len(src)] + copy(blk.literals, src) + return + } + + sLimit := int32(len(src)) - inputMargin + // stepSize is the number of bytes to skip on every main loop iteration. + // It should be >= 2. + const stepSize = 2 + + // TEMPLATE + const hashLog = tableBits + // seems global, but would be nice to tweak. + const kSearchStrength = 8 + + // nextEmit is where in src the next emitLiteral should start from. + nextEmit := s + cv := load6432(src, s) + + // Relative offsets + offset1 := int32(blk.recentOffsets[0]) + offset2 := int32(blk.recentOffsets[1]) + + addLiterals := func(s *seq, until int32) { + if until == nextEmit { + return + } + blk.literals = append(blk.literals, src[nextEmit:until]...) + s.litLen = uint32(until - nextEmit) + } + if debug { + println("recent offsets:", blk.recentOffsets) + } + +encodeLoop: + for { + // t will contain the match offset when we find one. + // When existing the search loop, we have already checked 4 bytes. + var t int32 + + // We will not use repeat offsets across blocks. + // By not using them for the first 3 matches + + for { + nextHash := hash6(cv, hashLog) + nextHash2 := hash6(cv>>8, hashLog) + candidate := e.table[nextHash] + candidate2 := e.table[nextHash2] + repIndex := s - offset1 + 2 + + e.table[nextHash] = tableEntry{offset: s + e.cur, val: uint32(cv)} + e.table[nextHash2] = tableEntry{offset: s + e.cur + 1, val: uint32(cv >> 8)} + + if len(blk.sequences) > 2 && load3232(src, repIndex) == uint32(cv>>16) { + // Consider history as well. + var seq seq + // lenght := 4 + e.matchlen(s+6, repIndex+4, src) + lenght := 4 + int32(matchLen(src[s+6:], src[repIndex+4:])) + + seq.matchLen = uint32(lenght - zstdMinMatch) + + // We might be able to match backwards. + // Extend as long as we can. + start := s + 2 + // We end the search early, so we don't risk 0 literals + // and have to do special offset treatment. + startLimit := nextEmit + 1 + + sMin := s - e.maxMatchOff + if sMin < 0 { + sMin = 0 + } + for repIndex > sMin && start > startLimit && src[repIndex-1] == src[start-1] { + repIndex-- + start-- + seq.matchLen++ + } + addLiterals(&seq, start) + + // rep 0 + seq.offset = 1 + if debugSequences { + println("repeat sequence", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + s += lenght + 2 + nextEmit = s + if s >= sLimit { + if debug { + println("repeat ended", s, lenght) + + } + break encodeLoop + } + cv = load6432(src, s) + continue + } + coffset0 := s - (candidate.offset - e.cur) + coffset1 := s - (candidate2.offset - e.cur) + 1 + if coffset0 < e.maxMatchOff && uint32(cv) == candidate.val { + // found a regular match + t = candidate.offset - e.cur + if debug && s <= t { + panic("s <= t") + } + if debug && s-t > e.maxMatchOff { + panic("s - t >e.maxMatchOff") + } + break + } + + if coffset1 < e.maxMatchOff && uint32(cv>>8) == candidate2.val { + // found a regular match + t = candidate2.offset - e.cur + s++ + if debug && s <= t { + panic("s <= t") + } + if debug && s-t > e.maxMatchOff { + panic("s - t >e.maxMatchOff") + } + if debug && t < 0 { + panic("t<0") + } + break + } + s += stepSize + ((s - nextEmit) >> (kSearchStrength - 1)) + if s >= sLimit { + break encodeLoop + } + cv = load6432(src, s) + } + // A 4-byte match has been found. We'll later see if more than 4 bytes. + offset2 = offset1 + offset1 = s - t + + if debug && s <= t { + panic("s <= t") + } + + // Extend the 4-byte match as long as possible. + //l := e.matchlenNoHist(s+4, t+4, src) + 4 + l := int32(matchLen(src[s+4:], src[t+4:])) + 4 + + // Extend backwards + tMin := s - e.maxMatchOff + if tMin < 0 { + tMin = 0 + } + for t > tMin && s > nextEmit && src[t-1] == src[s-1] { + s-- + t-- + l++ + } + + // Write our sequence. + var seq seq + seq.litLen = uint32(s - nextEmit) + seq.matchLen = uint32(l - zstdMinMatch) + if seq.litLen > 0 { + blk.literals = append(blk.literals, src[nextEmit:s]...) + } + // Don't use repeat offsets + seq.offset = uint32(s-t) + 3 + s += l + if debugSequences { + println("sequence", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + nextEmit = s + if s >= sLimit { + break encodeLoop + } + cv = load6432(src, s) + + // Check offset 2 + if o2 := s - offset2; len(blk.sequences) > 2 && load3232(src, o2) == uint32(cv) { + // We have at least 4 byte match. + // No need to check backwards. We come straight from a match + //l := 4 + e.matchlenNoHist(s+4, o2+4, src) + l := 4 + int32(matchLen(src[s+4:], src[o2+4:])) + + // Store this, since we have it. + nextHash := hash6(cv, hashLog) + e.table[nextHash] = tableEntry{offset: s + e.cur, val: uint32(cv)} + seq.matchLen = uint32(l) - zstdMinMatch + seq.litLen = 0 + // Since litlen is always 0, this is offset 1. + seq.offset = 1 + s += l + nextEmit = s + if debugSequences { + println("sequence", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + + // Swap offset 1 and 2. + offset1, offset2 = offset2, offset1 + if s >= sLimit { + break encodeLoop + } + // Prepare next loop. + cv = load6432(src, s) + } + } + + if int(nextEmit) < len(src) { + blk.literals = append(blk.literals, src[nextEmit:]...) + blk.extraLits = len(src) - int(nextEmit) + } + if debug { + println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) + } +} + +func (e *fastEncoder) addBlock(src []byte) int32 { + // check if we have space already + if len(e.hist)+len(src) > cap(e.hist) { + if cap(e.hist) == 0 { + l := e.maxMatchOff * 2 + // Make it at least 1MB. + if l < 1<<20 { + l = 1 << 20 + } + e.hist = make([]byte, 0, l) + } else { + if cap(e.hist) < int(e.maxMatchOff*2) { + panic("unexpected buffer size") + } + // Move down + offset := int32(len(e.hist)) - e.maxMatchOff + copy(e.hist[0:e.maxMatchOff], e.hist[offset:]) + e.cur += offset + e.hist = e.hist[:e.maxMatchOff] + } + } + s := int32(len(e.hist)) + e.hist = append(e.hist, src...) + return s +} + +// useBlock will replace the block with the provided one, +// but transfer recent offsets from the previous. +func (e *fastEncoder) UseBlock(enc *blockEnc) { + enc.reset(e.blk) + e.blk = enc +} + +func (e *fastEncoder) matchlenNoHist(s, t int32, src []byte) int32 { + // Extend the match to be as long as possible. + return int32(matchLen(src[s:], src[t:])) +} + +func (e *fastEncoder) matchlen(s, t int32, src []byte) int32 { + if debug { + if s < 0 { + panic("s<0") + } + if t < 0 { + panic("t<0") + } + if s-t > e.maxMatchOff { + panic(s - t) + } + } + s1 := int(s) + maxMatchLength - 4 + if s1 > len(src) { + s1 = len(src) + } + + // Extend the match to be as long as possible. + return int32(matchLen(src[s:s1], src[t:])) +} + +// Reset the encoding table. +func (e *fastEncoder) Reset() { + if e.blk == nil { + e.blk = &blockEnc{} + e.blk.init() + } else { + e.blk.reset(nil) + } + e.blk.initNewEncode() + if e.crc == nil { + e.crc = xxhash.New() + } else { + e.crc.Reset() + } + if cap(e.hist) < int(e.maxMatchOff*2) { + l := e.maxMatchOff * 2 + // Make it at least 1MB. + if l < 1<<20 { + l = 1 << 20 + } + e.hist = make([]byte, 0, l) + } + // We offset current position so everything will be out of reach + e.cur += e.maxMatchOff + int32(len(e.hist)) + e.hist = e.hist[:0] +} diff --git a/vendor/github.com/klauspost/compress/zstd/enc_params.go b/vendor/github.com/klauspost/compress/zstd/enc_params.go new file mode 100644 index 000000000..b6779ecb6 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/enc_params.go @@ -0,0 +1,154 @@ +// Copyright 2019+ Klaus Post. All rights reserved. +// License information can be found in the LICENSE file. +// Based on work by Yann Collet, released under BSD License. + +package zstd + +type encParams struct { + // largest match distance : larger == more compression, more memory needed during decompression + windowLog uint8 + + // fully searched segment : larger == more compression, slower, more memory (useless for fast) + chainLog uint8 + + // dispatch table : larger == faster, more memory + hashLog uint8 + + // < nb of searches : larger == more compression, slower + searchLog uint8 + + // < match length searched : larger == faster decompression, sometimes less compression + minMatch uint8 + + // acceptable match size for optimal parser (only) : larger == more compression, slower + targetLength uint32 + + // see ZSTD_strategy definition above + strategy strategy +} + +// strategy defines the algorithm to use when generating sequences. +type strategy uint8 + +const ( + // Compression strategies, listed from fastest to strongest + strategyFast strategy = iota + 1 + strategyDfast + strategyGreedy + strategyLazy + strategyLazy2 + strategyBtlazy2 + strategyBtopt + strategyBtultra + strategyBtultra2 + // note : new strategies _might_ be added in the future. + // Only the order (from fast to strong) is guaranteed + +) + +var defEncParams = [4][]encParams{ + { // "default" - for any srcSize > 256 KB + // W, C, H, S, L, TL, strat + {19, 12, 13, 1, 6, 1, strategyFast}, // base for negative levels + {19, 13, 14, 1, 7, 0, strategyFast}, // level 1 + {20, 15, 16, 1, 6, 0, strategyFast}, // level 2 + {21, 16, 17, 1, 5, 1, strategyDfast}, // level 3 + {21, 18, 18, 1, 5, 1, strategyDfast}, // level 4 + {21, 18, 19, 2, 5, 2, strategyGreedy}, // level 5 + {21, 19, 19, 3, 5, 4, strategyGreedy}, // level 6 + {21, 19, 19, 3, 5, 8, strategyLazy}, // level 7 + {21, 19, 19, 3, 5, 16, strategyLazy2}, // level 8 + {21, 19, 20, 4, 5, 16, strategyLazy2}, // level 9 + {22, 20, 21, 4, 5, 16, strategyLazy2}, // level 10 + {22, 21, 22, 4, 5, 16, strategyLazy2}, // level 11 + {22, 21, 22, 5, 5, 16, strategyLazy2}, // level 12 + {22, 21, 22, 5, 5, 32, strategyBtlazy2}, // level 13 + {22, 22, 23, 5, 5, 32, strategyBtlazy2}, // level 14 + {22, 23, 23, 6, 5, 32, strategyBtlazy2}, // level 15 + {22, 22, 22, 5, 5, 48, strategyBtopt}, // level 16 + {23, 23, 22, 5, 4, 64, strategyBtopt}, // level 17 + {23, 23, 22, 6, 3, 64, strategyBtultra}, // level 18 + {23, 24, 22, 7, 3, 256, strategyBtultra2}, // level 19 + {25, 25, 23, 7, 3, 256, strategyBtultra2}, // level 20 + {26, 26, 24, 7, 3, 512, strategyBtultra2}, // level 21 + {27, 27, 25, 9, 3, 999, strategyBtultra2}, // level 22 + }, + { // for srcSize <= 256 KB + // W, C, H, S, L, T, strat + {18, 12, 13, 1, 5, 1, strategyFast}, // base for negative levels + {18, 13, 14, 1, 6, 0, strategyFast}, // level 1 + {18, 14, 14, 1, 5, 1, strategyDfast}, // level 2 + {18, 16, 16, 1, 4, 1, strategyDfast}, // level 3 + {18, 16, 17, 2, 5, 2, strategyGreedy}, // level 4. + {18, 18, 18, 3, 5, 2, strategyGreedy}, // level 5. + {18, 18, 19, 3, 5, 4, strategyLazy}, // level 6. + {18, 18, 19, 4, 4, 4, strategyLazy}, // level 7 + {18, 18, 19, 4, 4, 8, strategyLazy2}, // level 8 + {18, 18, 19, 5, 4, 8, strategyLazy2}, // level 9 + {18, 18, 19, 6, 4, 8, strategyLazy2}, // level 10 + {18, 18, 19, 5, 4, 12, strategyBtlazy2}, // level 11. + {18, 19, 19, 7, 4, 12, strategyBtlazy2}, // level 12. + {18, 18, 19, 4, 4, 16, strategyBtopt}, // level 13 + {18, 18, 19, 4, 3, 32, strategyBtopt}, // level 14. + {18, 18, 19, 6, 3, 128, strategyBtopt}, // level 15. + {18, 19, 19, 6, 3, 128, strategyBtultra}, // level 16. + {18, 19, 19, 8, 3, 256, strategyBtultra}, // level 17. + {18, 19, 19, 6, 3, 128, strategyBtultra2}, // level 18. + {18, 19, 19, 8, 3, 256, strategyBtultra2}, // level 19. + {18, 19, 19, 10, 3, 512, strategyBtultra2}, // level 20. + {18, 19, 19, 12, 3, 512, strategyBtultra2}, // level 21. + {18, 19, 19, 13, 3, 999, strategyBtultra2}, // level 22. + }, + { // for srcSize <= 128 KB + // W, C, H, S, L, T, strat + {17, 12, 12, 1, 5, 1, strategyFast}, // base for negative levels + {17, 12, 13, 1, 6, 0, strategyFast}, // level 1 + {17, 13, 15, 1, 5, 0, strategyFast}, // level 2 + {17, 15, 16, 2, 5, 1, strategyDfast}, // level 3 + {17, 17, 17, 2, 4, 1, strategyDfast}, // level 4 + {17, 16, 17, 3, 4, 2, strategyGreedy}, // level 5 + {17, 17, 17, 3, 4, 4, strategyLazy}, // level 6 + {17, 17, 17, 3, 4, 8, strategyLazy2}, // level 7 + {17, 17, 17, 4, 4, 8, strategyLazy2}, // level 8 + {17, 17, 17, 5, 4, 8, strategyLazy2}, // level 9 + {17, 17, 17, 6, 4, 8, strategyLazy2}, // level 10 + {17, 17, 17, 5, 4, 8, strategyBtlazy2}, // level 11 + {17, 18, 17, 7, 4, 12, strategyBtlazy2}, // level 12 + {17, 18, 17, 3, 4, 12, strategyBtopt}, // level 13. + {17, 18, 17, 4, 3, 32, strategyBtopt}, // level 14. + {17, 18, 17, 6, 3, 256, strategyBtopt}, // level 15. + {17, 18, 17, 6, 3, 128, strategyBtultra}, // level 16. + {17, 18, 17, 8, 3, 256, strategyBtultra}, // level 17. + {17, 18, 17, 10, 3, 512, strategyBtultra}, // level 18. + {17, 18, 17, 5, 3, 256, strategyBtultra2}, // level 19. + {17, 18, 17, 7, 3, 512, strategyBtultra2}, // level 20. + {17, 18, 17, 9, 3, 512, strategyBtultra2}, // level 21. + {17, 18, 17, 11, 3, 999, strategyBtultra2}, // level 22. + }, + { // for srcSize <= 16 KB + // W, C, H, S, L, T, strat + {14, 12, 13, 1, 5, 1, strategyFast}, // base for negative levels + {14, 14, 15, 1, 5, 0, strategyFast}, // level 1 + {14, 14, 15, 1, 4, 0, strategyFast}, // level 2 + {14, 14, 15, 2, 4, 1, strategyDfast}, // level 3 + {14, 14, 14, 4, 4, 2, strategyGreedy}, // level 4 + {14, 14, 14, 3, 4, 4, strategyLazy}, // level 5. + {14, 14, 14, 4, 4, 8, strategyLazy2}, // level 6 + {14, 14, 14, 6, 4, 8, strategyLazy2}, // level 7 + {14, 14, 14, 8, 4, 8, strategyLazy2}, // level 8. + {14, 15, 14, 5, 4, 8, strategyBtlazy2}, // level 9. + {14, 15, 14, 9, 4, 8, strategyBtlazy2}, // level 10. + {14, 15, 14, 3, 4, 12, strategyBtopt}, // level 11. + {14, 15, 14, 4, 3, 24, strategyBtopt}, // level 12. + {14, 15, 14, 5, 3, 32, strategyBtultra}, // level 13. + {14, 15, 15, 6, 3, 64, strategyBtultra}, // level 14. + {14, 15, 15, 7, 3, 256, strategyBtultra}, // level 15. + {14, 15, 15, 5, 3, 48, strategyBtultra2}, // level 16. + {14, 15, 15, 6, 3, 128, strategyBtultra2}, // level 17. + {14, 15, 15, 7, 3, 256, strategyBtultra2}, // level 18. + {14, 15, 15, 8, 3, 256, strategyBtultra2}, // level 19. + {14, 15, 15, 8, 3, 512, strategyBtultra2}, // level 20. + {14, 15, 15, 9, 3, 512, strategyBtultra2}, // level 21. + {14, 15, 15, 10, 3, 999, strategyBtultra2}, // level 22. + }, +} diff --git a/vendor/github.com/klauspost/compress/zstd/encoder.go b/vendor/github.com/klauspost/compress/zstd/encoder.go new file mode 100644 index 000000000..366dd66bd --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/encoder.go @@ -0,0 +1,539 @@ +// Copyright 2019+ Klaus Post. All rights reserved. +// License information can be found in the LICENSE file. +// Based on work by Yann Collet, released under BSD License. + +package zstd + +import ( + "crypto/rand" + "fmt" + "io" + rdebug "runtime/debug" + "sync" + + "github.com/klauspost/compress/zstd/internal/xxhash" +) + +// Encoder provides encoding to Zstandard. +// An Encoder can be used for either compressing a stream via the +// io.WriteCloser interface supported by the Encoder or as multiple independent +// tasks via the EncodeAll function. +// Smaller encodes are encouraged to use the EncodeAll function. +// Use NewWriter to create a new instance. +type Encoder struct { + o encoderOptions + encoders chan encoder + state encoderState + init sync.Once +} + +type encoder interface { + Encode(blk *blockEnc, src []byte) + EncodeNoHist(blk *blockEnc, src []byte) + Block() *blockEnc + CRC() *xxhash.Digest + AppendCRC([]byte) []byte + WindowSize(size int) int32 + UseBlock(*blockEnc) + Reset() +} + +type encoderState struct { + w io.Writer + filling []byte + current []byte + previous []byte + encoder encoder + writing *blockEnc + err error + writeErr error + nWritten int64 + headerWritten bool + eofWritten bool + + // This waitgroup indicates an encode is running. + wg sync.WaitGroup + // This waitgroup indicates we have a block encoding/writing. + wWg sync.WaitGroup +} + +// NewWriter will create a new Zstandard encoder. +// If the encoder will be used for encoding blocks a nil writer can be used. +func NewWriter(w io.Writer, opts ...EOption) (*Encoder, error) { + initPredefined() + var e Encoder + e.o.setDefault() + for _, o := range opts { + err := o(&e.o) + if err != nil { + return nil, err + } + } + if w != nil { + e.Reset(w) + } else { + e.init.Do(func() { + e.initialize() + }) + } + return &e, nil +} + +func (e *Encoder) initialize() { + e.encoders = make(chan encoder, e.o.concurrent) + for i := 0; i < e.o.concurrent; i++ { + e.encoders <- e.o.encoder() + } +} + +// Reset will re-initialize the writer and new writes will encode to the supplied writer +// as a new, independent stream. +func (e *Encoder) Reset(w io.Writer) { + e.init.Do(func() { + e.initialize() + }) + s := &e.state + s.wg.Wait() + s.wWg.Wait() + if cap(s.filling) == 0 { + s.filling = make([]byte, 0, e.o.blockSize) + } + if cap(s.current) == 0 { + s.current = make([]byte, 0, e.o.blockSize) + } + if cap(s.previous) == 0 { + s.previous = make([]byte, 0, e.o.blockSize) + } + if s.encoder == nil { + s.encoder = e.o.encoder() + } + if s.writing == nil { + s.writing = &blockEnc{} + s.writing.init() + } + s.writing.initNewEncode() + s.filling = s.filling[:0] + s.current = s.current[:0] + s.previous = s.previous[:0] + s.encoder.Reset() + s.headerWritten = false + s.eofWritten = false + s.w = w + s.err = nil + s.nWritten = 0 + s.writeErr = nil +} + +// Write data to the encoder. +// Input data will be buffered and as the buffer fills up +// content will be compressed and written to the output. +// When done writing, use Close to flush the remaining output +// and write CRC if requested. +func (e *Encoder) Write(p []byte) (n int, err error) { + s := &e.state + for len(p) > 0 { + if len(p)+len(s.filling) < e.o.blockSize { + if e.o.crc { + _, _ = s.encoder.CRC().Write(p) + } + s.filling = append(s.filling, p...) + return n + len(p), nil + } + add := p + if len(p)+len(s.filling) > e.o.blockSize { + add = add[:e.o.blockSize-len(s.filling)] + } + if e.o.crc { + _, _ = s.encoder.CRC().Write(add) + } + s.filling = append(s.filling, add...) + p = p[len(add):] + n += len(add) + if len(s.filling) < e.o.blockSize { + return n, nil + } + err := e.nextBlock(false) + if err != nil { + return n, err + } + if debug && len(s.filling) > 0 { + panic(len(s.filling)) + } + } + return n, nil +} + +// nextBlock will synchronize and start compressing input in e.state.filling. +// If an error has occurred during encoding it will be returned. +func (e *Encoder) nextBlock(final bool) error { + s := &e.state + // Wait for current block. + s.wg.Wait() + if s.err != nil { + return s.err + } + if len(s.filling) > e.o.blockSize { + return fmt.Errorf("block > maxStoreBlockSize") + } + if !s.headerWritten { + var tmp [maxHeaderSize]byte + fh := frameHeader{ + ContentSize: 0, + WindowSize: uint32(s.encoder.WindowSize(0)), + SingleSegment: false, + Checksum: e.o.crc, + DictID: 0, + } + dst, err := fh.appendTo(tmp[:0]) + if err != nil { + return err + } + s.headerWritten = true + s.wWg.Wait() + var n2 int + n2, s.err = s.w.Write(dst) + if s.err != nil { + return s.err + } + s.nWritten += int64(n2) + } + if s.eofWritten { + // Ensure we only write it once. + final = false + } + + if len(s.filling) == 0 { + // Final block, but no data. + if final { + enc := s.encoder + blk := enc.Block() + blk.reset(nil) + blk.last = true + blk.encodeRaw(nil) + s.wWg.Wait() + _, s.err = s.w.Write(blk.output) + s.nWritten += int64(len(blk.output)) + s.eofWritten = true + } + return s.err + } + + // Move blocks forward. + s.filling, s.current, s.previous = s.previous[:0], s.filling, s.current + s.wg.Add(1) + go func(src []byte) { + if debug { + println("Adding block,", len(src), "bytes, final:", final) + } + defer func() { + if r := recover(); r != nil { + s.err = fmt.Errorf("panic while encoding: %v", r) + rdebug.PrintStack() + } + s.wg.Done() + }() + enc := s.encoder + blk := enc.Block() + enc.Encode(blk, src) + blk.last = final + if final { + s.eofWritten = true + } + // Wait for pending writes. + s.wWg.Wait() + if s.writeErr != nil { + s.err = s.writeErr + return + } + // Transfer encoders from previous write block. + blk.swapEncoders(s.writing) + // Transfer recent offsets to next. + enc.UseBlock(s.writing) + s.writing = blk + s.wWg.Add(1) + go func() { + defer func() { + if r := recover(); r != nil { + s.writeErr = fmt.Errorf("panic while encoding/writing: %v", r) + rdebug.PrintStack() + } + s.wWg.Done() + }() + err := errIncompressible + // If we got the exact same number of literals as input, + // assume the literals cannot be compressed. + if len(src) != len(blk.literals) || len(src) != e.o.blockSize { + err = blk.encode(e.o.noEntropy) + } + switch err { + case errIncompressible: + if debug { + println("Storing incompressible block as raw") + } + blk.encodeRaw(src) + // In fast mode, we do not transfer offsets, so we don't have to deal with changing the. + case nil: + default: + s.writeErr = err + return + } + _, s.writeErr = s.w.Write(blk.output) + s.nWritten += int64(len(blk.output)) + }() + }(s.current) + return nil +} + +// ReadFrom reads data from r until EOF or error. +// The return value n is the number of bytes read. +// Any error except io.EOF encountered during the read is also returned. +// +// The Copy function uses ReaderFrom if available. +func (e *Encoder) ReadFrom(r io.Reader) (n int64, err error) { + if debug { + println("Using ReadFrom") + } + // Maybe handle stuff queued? + e.state.filling = e.state.filling[:e.o.blockSize] + src := e.state.filling + for { + n2, err := r.Read(src) + _, _ = e.state.encoder.CRC().Write(src[:n2]) + // src is now the unfilled part... + src = src[n2:] + n += int64(n2) + switch err { + case io.EOF: + e.state.filling = e.state.filling[:len(e.state.filling)-len(src)] + if debug { + println("ReadFrom: got EOF final block:", len(e.state.filling)) + } + return n, e.nextBlock(true) + default: + if debug { + println("ReadFrom: got error:", err) + } + e.state.err = err + return n, err + case nil: + } + if len(src) > 0 { + if debug { + println("ReadFrom: got space left in source:", len(src)) + } + continue + } + err = e.nextBlock(false) + if err != nil { + return n, err + } + e.state.filling = e.state.filling[:e.o.blockSize] + src = e.state.filling + } +} + +// Flush will send the currently written data to output +// and block until everything has been written. +// This should only be used on rare occasions where pushing the currently queued data is critical. +func (e *Encoder) Flush() error { + s := &e.state + if len(s.filling) > 0 { + err := e.nextBlock(false) + if err != nil { + return err + } + } + s.wg.Wait() + s.wWg.Wait() + if s.err != nil { + return s.err + } + return s.writeErr +} + +// Close will flush the final output and close the stream. +// The function will block until everything has been written. +// The Encoder can still be re-used after calling this. +func (e *Encoder) Close() error { + s := &e.state + if s.encoder == nil { + return nil + } + err := e.nextBlock(true) + if err != nil { + return err + } + s.wg.Wait() + s.wWg.Wait() + + if s.err != nil { + return s.err + } + if s.writeErr != nil { + return s.writeErr + } + + // Write CRC + if e.o.crc && s.err == nil { + // heap alloc. + var tmp [4]byte + _, s.err = s.w.Write(s.encoder.AppendCRC(tmp[:0])) + s.nWritten += 4 + } + + // Add padding with content from crypto/rand.Reader + if s.err == nil && e.o.pad > 0 { + add := calcSkippableFrame(s.nWritten, int64(e.o.pad)) + frame, err := skippableFrame(s.filling[:0], add, rand.Reader) + if err != nil { + return err + } + _, s.err = s.w.Write(frame) + } + return s.err +} + +// EncodeAll will encode all input in src and append it to dst. +// This function can be called concurrently, but each call will only run on a single goroutine. +// If empty input is given, nothing is returned, unless WithZeroFrames is specified. +// Encoded blocks can be concatenated and the result will be the combined input stream. +// Data compressed with EncodeAll can be decoded with the Decoder, +// using either a stream or DecodeAll. +func (e *Encoder) EncodeAll(src, dst []byte) []byte { + if len(src) == 0 { + if e.o.fullZero { + // Add frame header. + fh := frameHeader{ + ContentSize: 0, + WindowSize: MinWindowSize, + SingleSegment: true, + // Adding a checksum would be a waste of space. + Checksum: false, + DictID: 0, + } + dst, _ = fh.appendTo(dst) + + // Write raw block as last one only. + var blk blockHeader + blk.setSize(0) + blk.setType(blockTypeRaw) + blk.setLast(true) + dst = blk.appendTo(dst) + } + return dst + } + e.init.Do(func() { + e.o.setDefault() + e.initialize() + }) + enc := <-e.encoders + defer func() { + // Release encoder reference to last block. + enc.Reset() + e.encoders <- enc + }() + enc.Reset() + blk := enc.Block() + // Use single segments when above minimum window and below 1MB. + single := len(src) < 1<<20 && len(src) > MinWindowSize + if e.o.single != nil { + single = *e.o.single + } + fh := frameHeader{ + ContentSize: uint64(len(src)), + WindowSize: uint32(enc.WindowSize(len(src))), + SingleSegment: single, + Checksum: e.o.crc, + DictID: 0, + } + + // If less than 1MB, allocate a buffer up front. + if len(dst) == 0 && cap(dst) == 0 && len(src) < 1<<20 { + dst = make([]byte, 0, len(src)) + } + dst, err := fh.appendTo(dst) + if err != nil { + panic(err) + } + + if len(src) <= e.o.blockSize && len(src) <= maxBlockSize { + // Slightly faster with no history and everything in one block. + if e.o.crc { + _, _ = enc.CRC().Write(src) + } + blk.reset(nil) + blk.last = true + enc.EncodeNoHist(blk, src) + + // If we got the exact same number of literals as input, + // assume the literals cannot be compressed. + err := errIncompressible + oldout := blk.output + if len(blk.literals) != len(src) || len(src) != e.o.blockSize { + // Output directly to dst + blk.output = dst + err = blk.encode(e.o.noEntropy) + } + + switch err { + case errIncompressible: + if debug { + println("Storing incompressible block as raw") + } + dst = blk.encodeRawTo(dst, src) + case nil: + dst = blk.output + default: + panic(err) + } + blk.output = oldout + } else { + for len(src) > 0 { + todo := src + if len(todo) > e.o.blockSize { + todo = todo[:e.o.blockSize] + } + src = src[len(todo):] + if e.o.crc { + _, _ = enc.CRC().Write(todo) + } + blk.reset(nil) + blk.pushOffsets() + enc.Encode(blk, todo) + if len(src) == 0 { + blk.last = true + } + err := errIncompressible + // If we got the exact same number of literals as input, + // assume the literals cannot be compressed. + if len(blk.literals) != len(todo) || len(todo) != e.o.blockSize { + err = blk.encode(e.o.noEntropy) + } + + switch err { + case errIncompressible: + if debug { + println("Storing incompressible block as raw") + } + dst = blk.encodeRawTo(dst, todo) + blk.popOffsets() + case nil: + dst = append(dst, blk.output...) + default: + panic(err) + } + } + } + if e.o.crc { + dst = enc.AppendCRC(dst) + } + // Add padding with content from crypto/rand.Reader + if e.o.pad > 0 { + add := calcSkippableFrame(int64(len(dst)), int64(e.o.pad)) + dst, err = skippableFrame(dst, add, rand.Reader) + if err != nil { + panic(err) + } + } + return dst +} diff --git a/vendor/github.com/klauspost/compress/zstd/encoder_options.go b/vendor/github.com/klauspost/compress/zstd/encoder_options.go new file mode 100644 index 000000000..40eb45733 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/encoder_options.go @@ -0,0 +1,231 @@ +package zstd + +import ( + "errors" + "fmt" + "runtime" + "strings" +) + +// EOption is an option for creating a encoder. +type EOption func(*encoderOptions) error + +// options retains accumulated state of multiple options. +type encoderOptions struct { + concurrent int + crc bool + single *bool + pad int + blockSize int + windowSize int + level EncoderLevel + fullZero bool + noEntropy bool +} + +func (o *encoderOptions) setDefault() { + *o = encoderOptions{ + // use less ram: true for now, but may change. + concurrent: runtime.GOMAXPROCS(0), + crc: true, + single: nil, + blockSize: 1 << 16, + windowSize: 1 << 22, + level: SpeedDefault, + } +} + +// encoder returns an encoder with the selected options. +func (o encoderOptions) encoder() encoder { + switch o.level { + case SpeedDefault: + return &doubleFastEncoder{fastEncoder: fastEncoder{maxMatchOff: int32(o.windowSize)}} + case SpeedFastest: + return &fastEncoder{maxMatchOff: int32(o.windowSize)} + } + panic("unknown compression level") +} + +// WithEncoderCRC will add CRC value to output. +// Output will be 4 bytes larger. +func WithEncoderCRC(b bool) EOption { + return func(o *encoderOptions) error { o.crc = b; return nil } +} + +// WithEncoderConcurrency will set the concurrency, +// meaning the maximum number of decoders to run concurrently. +// The value supplied must be at least 1. +// By default this will be set to GOMAXPROCS. +func WithEncoderConcurrency(n int) EOption { + return func(o *encoderOptions) error { + if n <= 0 { + return fmt.Errorf("concurrency must be at least 1") + } + o.concurrent = n + return nil + } +} + +// WithWindowSize will set the maximum allowed back-reference distance. +// The value must be a power of two between WindowSizeMin and WindowSizeMax. +// A larger value will enable better compression but allocate more memory and, +// for above-default values, take considerably longer. +// The default value is determined by the compression level. +func WithWindowSize(n int) EOption { + return func(o *encoderOptions) error { + switch { + case n < MinWindowSize: + return fmt.Errorf("window size must be at least %d", MinWindowSize) + case n > MaxWindowSize: + return fmt.Errorf("window size must be at most %d", MaxWindowSize) + case (n & (n - 1)) != 0: + return errors.New("window size must be a power of 2") + } + + o.windowSize = n + if o.blockSize > o.windowSize { + o.blockSize = o.windowSize + } + return nil + } +} + +// WithEncoderPadding will add padding to all output so the size will be a multiple of n. +// This can be used to obfuscate the exact output size or make blocks of a certain size. +// The contents will be a skippable frame, so it will be invisible by the decoder. +// n must be > 0 and <= 1GB, 1<<30 bytes. +// The padded area will be filled with data from crypto/rand.Reader. +// If `EncodeAll` is used with data already in the destination, the total size will be multiple of this. +func WithEncoderPadding(n int) EOption { + return func(o *encoderOptions) error { + if n <= 0 { + return fmt.Errorf("padding must be at least 1") + } + // No need to waste our time. + if n == 1 { + o.pad = 0 + } + if n > 1<<30 { + return fmt.Errorf("padding must less than 1GB (1<<30 bytes) ") + } + o.pad = n + return nil + } +} + +// EncoderLevel predefines encoder compression levels. +// Only use the constants made available, since the actual mapping +// of these values are very likely to change and your compression could change +// unpredictably when upgrading the library. +type EncoderLevel int + +const ( + speedNotSet EncoderLevel = iota + + // SpeedFastest will choose the fastest reasonable compression. + // This is roughly equivalent to the fastest Zstandard mode. + SpeedFastest + + // SpeedDefault is the default "pretty fast" compression option. + // This is roughly equivalent to the default Zstandard mode (level 3). + SpeedDefault + + // speedLast should be kept as the last actual compression option. + // The is not for external usage, but is used to keep track of the valid options. + speedLast + + // SpeedBetterCompression will (in the future) yield better compression than the default, + // but at approximately 4x the CPU usage of the default. + // For now this is not implemented. + SpeedBetterCompression = SpeedDefault + + // SpeedBestCompression will choose the best available compression option. + // For now this is not implemented. + SpeedBestCompression = SpeedDefault +) + +// EncoderLevelFromString will convert a string representation of an encoding level back +// to a compression level. The compare is not case sensitive. +// If the string wasn't recognized, (false, SpeedDefault) will be returned. +func EncoderLevelFromString(s string) (bool, EncoderLevel) { + for l := EncoderLevel(speedNotSet + 1); l < speedLast; l++ { + if strings.EqualFold(s, l.String()) { + return true, l + } + } + return false, SpeedDefault +} + +// EncoderLevelFromZstd will return an encoder level that closest matches the compression +// ratio of a specific zstd compression level. +// Many input values will provide the same compression level. +func EncoderLevelFromZstd(level int) EncoderLevel { + switch { + case level < 3: + return SpeedFastest + case level >= 3: + return SpeedDefault + } + return SpeedDefault +} + +// String provides a string representation of the compression level. +func (e EncoderLevel) String() string { + switch e { + case SpeedFastest: + return "fastest" + case SpeedDefault: + return "default" + default: + return "invalid" + } +} + +// WithEncoderLevel specifies a predefined compression level. +func WithEncoderLevel(l EncoderLevel) EOption { + return func(o *encoderOptions) error { + switch { + case l <= speedNotSet || l >= speedLast: + return fmt.Errorf("unknown encoder level") + } + o.level = l + return nil + } +} + +// WithZeroFrames will encode 0 length input as full frames. +// This can be needed for compatibility with zstandard usage, +// but is not needed for this package. +func WithZeroFrames(b bool) EOption { + return func(o *encoderOptions) error { + o.fullZero = b + return nil + } +} + +// WithNoEntropyCompression will always skip entropy compression of literals. +// This can be useful if content has matches, but unlikely to benefit from entropy +// compression. Usually the slight speed improvement is not worth enabling this. +func WithNoEntropyCompression(b bool) EOption { + return func(o *encoderOptions) error { + o.noEntropy = b + return nil + } +} + +// WithSingleSegment will set the "single segment" flag when EncodeAll is used. +// If this flag is set, data must be regenerated within a single continuous memory segment. +// In this case, Window_Descriptor byte is skipped, but Frame_Content_Size is necessarily present. +// As a consequence, the decoder must allocate a memory segment of size equal or larger than size of your content. +// In order to preserve the decoder from unreasonable memory requirements, +// a decoder is allowed to reject a compressed frame which requests a memory size beyond decoder's authorized range. +// For broader compatibility, decoders are recommended to support memory sizes of at least 8 MB. +// This is only a recommendation, each decoder is free to support higher or lower limits, depending on local limitations. +// If this is not specified, block encodes will automatically choose this based on the input size. +// This setting has no effect on streamed encodes. +func WithSingleSegment(b bool) EOption { + return func(o *encoderOptions) error { + o.single = &b + return nil + } +} diff --git a/vendor/github.com/klauspost/compress/zstd/framedec.go b/vendor/github.com/klauspost/compress/zstd/framedec.go new file mode 100644 index 000000000..40790747a --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/framedec.go @@ -0,0 +1,489 @@ +// Copyright 2019+ Klaus Post. All rights reserved. +// License information can be found in the LICENSE file. +// Based on work by Yann Collet, released under BSD License. + +package zstd + +import ( + "bytes" + "encoding/hex" + "errors" + "hash" + "io" + "sync" + + "github.com/klauspost/compress/zstd/internal/xxhash" +) + +type frameDec struct { + o decoderOptions + crc hash.Hash64 + frameDone sync.WaitGroup + offset int64 + + WindowSize uint64 + DictionaryID uint32 + FrameContentSize uint64 + HasCheckSum bool + SingleSegment bool + + // maxWindowSize is the maximum windows size to support. + // should never be bigger than max-int. + maxWindowSize uint64 + + // In order queue of blocks being decoded. + decoding chan *blockDec + + // Frame history passed between blocks + history history + + rawInput byteBuffer + + // Byte buffer that can be reused for small input blocks. + bBuf byteBuf + + // asyncRunning indicates whether the async routine processes input on 'decoding'. + asyncRunning bool + asyncRunningMu sync.Mutex +} + +const ( + // The minimum Window_Size is 1 KB. + MinWindowSize = 1 << 10 + MaxWindowSize = 1 << 30 +) + +var ( + frameMagic = []byte{0x28, 0xb5, 0x2f, 0xfd} + skippableFrameMagic = []byte{0x2a, 0x4d, 0x18} +) + +func newFrameDec(o decoderOptions) *frameDec { + d := frameDec{ + o: o, + maxWindowSize: MaxWindowSize, + } + if d.maxWindowSize > o.maxDecodedSize { + d.maxWindowSize = o.maxDecodedSize + } + return &d +} + +// reset will read the frame header and prepare for block decoding. +// If nothing can be read from the input, io.EOF will be returned. +// Any other error indicated that the stream contained data, but +// there was a problem. +func (d *frameDec) reset(br byteBuffer) error { + d.HasCheckSum = false + d.WindowSize = 0 + var b []byte + for { + b = br.readSmall(4) + if b == nil { + return io.EOF + } + if !bytes.Equal(b[1:4], skippableFrameMagic) || b[0]&0xf0 != 0x50 { + if debug { + println("Not skippable", hex.EncodeToString(b), hex.EncodeToString(skippableFrameMagic)) + } + // Break if not skippable frame. + break + } + // Read size to skip + b = br.readSmall(4) + if b == nil { + println("Reading Frame Size EOF") + return io.ErrUnexpectedEOF + } + n := uint32(b[0]) | (uint32(b[1]) << 8) | (uint32(b[2]) << 16) | (uint32(b[3]) << 24) + println("Skipping frame with", n, "bytes.") + err := br.skipN(int(n)) + if err != nil { + if debug { + println("Reading discarded frame", err) + } + return err + } + } + if !bytes.Equal(b, frameMagic) { + println("Got magic numbers: ", b, "want:", frameMagic) + return ErrMagicMismatch + } + + // Read Frame_Header_Descriptor + fhd, err := br.readByte() + if err != nil { + println("Reading Frame_Header_Descriptor", err) + return err + } + d.SingleSegment = fhd&(1<<5) != 0 + + if fhd&(1<<3) != 0 { + return errors.New("Reserved bit set on frame header") + } + + // Read Window_Descriptor + // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#window_descriptor + d.WindowSize = 0 + if !d.SingleSegment { + wd, err := br.readByte() + if err != nil { + println("Reading Window_Descriptor", err) + return err + } + printf("raw: %x, mantissa: %d, exponent: %d\n", wd, wd&7, wd>>3) + windowLog := 10 + (wd >> 3) + windowBase := uint64(1) << windowLog + windowAdd := (windowBase / 8) * uint64(wd&0x7) + d.WindowSize = windowBase + windowAdd + } + + // Read Dictionary_ID + // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary_id + d.DictionaryID = 0 + if size := fhd & 3; size != 0 { + if size == 3 { + size = 4 + } + b = br.readSmall(int(size)) + if b == nil { + if debug { + println("Reading Dictionary_ID", io.ErrUnexpectedEOF) + } + return io.ErrUnexpectedEOF + } + switch size { + case 1: + d.DictionaryID = uint32(b[0]) + case 2: + d.DictionaryID = uint32(b[0]) | (uint32(b[1]) << 8) + case 4: + d.DictionaryID = uint32(b[0]) | (uint32(b[1]) << 8) | (uint32(b[2]) << 16) | (uint32(b[3]) << 24) + } + if debug { + println("Dict size", size, "ID:", d.DictionaryID) + } + if d.DictionaryID != 0 { + return ErrUnknownDictionary + } + } + + // Read Frame_Content_Size + // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#frame_content_size + var fcsSize int + v := fhd >> 6 + switch v { + case 0: + if d.SingleSegment { + fcsSize = 1 + } + default: + fcsSize = 1 << v + } + d.FrameContentSize = 0 + if fcsSize > 0 { + b := br.readSmall(fcsSize) + if b == nil { + println("Reading Frame content", io.ErrUnexpectedEOF) + return io.ErrUnexpectedEOF + } + switch fcsSize { + case 1: + d.FrameContentSize = uint64(b[0]) + case 2: + // When FCS_Field_Size is 2, the offset of 256 is added. + d.FrameContentSize = uint64(b[0]) | (uint64(b[1]) << 8) + 256 + case 4: + d.FrameContentSize = uint64(b[0]) | (uint64(b[1]) << 8) | (uint64(b[2]) << 16) | (uint64(b[3]) << 24) + case 8: + d1 := uint32(b[0]) | (uint32(b[1]) << 8) | (uint32(b[2]) << 16) | (uint32(b[3]) << 24) + d2 := uint32(b[4]) | (uint32(b[5]) << 8) | (uint32(b[6]) << 16) | (uint32(b[7]) << 24) + d.FrameContentSize = uint64(d1) | (uint64(d2) << 32) + } + if debug { + println("field size bits:", v, "fcsSize:", fcsSize, "FrameContentSize:", d.FrameContentSize, hex.EncodeToString(b[:fcsSize]), "singleseg:", d.SingleSegment, "window:", d.WindowSize) + } + } + // Move this to shared. + d.HasCheckSum = fhd&(1<<2) != 0 + if d.HasCheckSum { + if d.crc == nil { + d.crc = xxhash.New() + } + d.crc.Reset() + } + + if d.WindowSize == 0 && d.SingleSegment { + // We may not need window in this case. + d.WindowSize = d.FrameContentSize + if d.WindowSize < MinWindowSize { + d.WindowSize = MinWindowSize + } + } + + if d.WindowSize > d.maxWindowSize { + printf("window size %d > max %d\n", d.WindowSize, d.maxWindowSize) + return ErrWindowSizeExceeded + } + // The minimum Window_Size is 1 KB. + if d.WindowSize < MinWindowSize { + println("got window size: ", d.WindowSize) + return ErrWindowSizeTooSmall + } + d.history.windowSize = int(d.WindowSize) + d.history.maxSize = d.history.windowSize + maxBlockSize + // history contains input - maybe we do something + d.rawInput = br + return nil +} + +// next will start decoding the next block from stream. +func (d *frameDec) next(block *blockDec) error { + if debug { + printf("decoding new block %p:%p", block, block.data) + } + err := block.reset(d.rawInput, d.WindowSize) + if err != nil { + println("block error:", err) + // Signal the frame decoder we have a problem. + d.sendErr(block, err) + return err + } + block.input <- struct{}{} + if debug { + println("next block:", block) + } + d.asyncRunningMu.Lock() + defer d.asyncRunningMu.Unlock() + if !d.asyncRunning { + return nil + } + if block.Last { + // We indicate the frame is done by sending io.EOF + d.decoding <- block + return io.EOF + } + d.decoding <- block + return nil +} + +// sendEOF will queue an error block on the frame. +// This will cause the frame decoder to return when it encounters the block. +// Returns true if the decoder was added. +func (d *frameDec) sendErr(block *blockDec, err error) bool { + d.asyncRunningMu.Lock() + defer d.asyncRunningMu.Unlock() + if !d.asyncRunning { + return false + } + + println("sending error", err.Error()) + block.sendErr(err) + d.decoding <- block + return true +} + +// checkCRC will check the checksum if the frame has one. +// Will return ErrCRCMismatch if crc check failed, otherwise nil. +func (d *frameDec) checkCRC() error { + if !d.HasCheckSum { + return nil + } + var tmp [4]byte + got := d.crc.Sum64() + // Flip to match file order. + tmp[0] = byte(got >> 0) + tmp[1] = byte(got >> 8) + tmp[2] = byte(got >> 16) + tmp[3] = byte(got >> 24) + + // We can overwrite upper tmp now + want := d.rawInput.readSmall(4) + if want == nil { + println("CRC missing?") + return io.ErrUnexpectedEOF + } + + if !bytes.Equal(tmp[:], want) { + if debug { + println("CRC Check Failed:", tmp[:], "!=", want) + } + return ErrCRCMismatch + } + if debug { + println("CRC ok", tmp[:]) + } + return nil +} + +func (d *frameDec) initAsync() { + if !d.o.lowMem && !d.SingleSegment { + // set max extra size history to 20MB. + d.history.maxSize = d.history.windowSize + maxBlockSize*10 + } + // re-alloc if more than one extra block size. + if d.o.lowMem && cap(d.history.b) > d.history.maxSize+maxBlockSize { + d.history.b = make([]byte, 0, d.history.maxSize) + } + if cap(d.history.b) < d.history.maxSize { + d.history.b = make([]byte, 0, d.history.maxSize) + } + if cap(d.decoding) < d.o.concurrent { + d.decoding = make(chan *blockDec, d.o.concurrent) + } + if debug { + h := d.history + printf("history init. len: %d, cap: %d", len(h.b), cap(h.b)) + } + d.asyncRunningMu.Lock() + d.asyncRunning = true + d.asyncRunningMu.Unlock() +} + +// startDecoder will start decoding blocks and write them to the writer. +// The decoder will stop as soon as an error occurs or at end of frame. +// When the frame has finished decoding the *bufio.Reader +// containing the remaining input will be sent on frameDec.frameDone. +func (d *frameDec) startDecoder(output chan decodeOutput) { + // TODO: Init to dictionary + d.history.reset() + written := int64(0) + + defer func() { + d.asyncRunningMu.Lock() + d.asyncRunning = false + d.asyncRunningMu.Unlock() + + // Drain the currently decoding. + d.history.error = true + flushdone: + for { + select { + case b := <-d.decoding: + b.history <- &d.history + output <- <-b.result + default: + break flushdone + } + } + println("frame decoder done, signalling done") + d.frameDone.Done() + }() + // Get decoder for first block. + block := <-d.decoding + block.history <- &d.history + for { + var next *blockDec + // Get result + r := <-block.result + if r.err != nil { + println("Result contained error", r.err) + output <- r + return + } + if debug { + println("got result, from ", d.offset, "to", d.offset+int64(len(r.b))) + d.offset += int64(len(r.b)) + } + if !block.Last { + // Send history to next block + select { + case next = <-d.decoding: + if debug { + println("Sending ", len(d.history.b), "bytes as history") + } + next.history <- &d.history + default: + // Wait until we have sent the block, so + // other decoders can potentially get the decoder. + next = nil + } + } + + // Add checksum, async to decoding. + if d.HasCheckSum { + n, err := d.crc.Write(r.b) + if err != nil { + r.err = err + if n != len(r.b) { + r.err = io.ErrShortWrite + } + output <- r + return + } + } + written += int64(len(r.b)) + if d.SingleSegment && uint64(written) > d.FrameContentSize { + println("runDecoder: single segment and", uint64(written), ">", d.FrameContentSize) + r.err = ErrFrameSizeExceeded + output <- r + return + } + if block.Last { + r.err = d.checkCRC() + output <- r + return + } + output <- r + if next == nil { + // There was no decoder available, we wait for one now that we have sent to the writer. + if debug { + println("Sending ", len(d.history.b), " bytes as history") + } + next = <-d.decoding + next.history <- &d.history + } + block = next + } +} + +// runDecoder will create a sync decoder that will decode a block of data. +func (d *frameDec) runDecoder(dst []byte, dec *blockDec) ([]byte, error) { + // TODO: Init to dictionary + d.history.reset() + saved := d.history.b + + // We use the history for output to avoid copying it. + d.history.b = dst + // Store input length, so we only check new data. + crcStart := len(dst) + var err error + for { + err = dec.reset(d.rawInput, d.WindowSize) + if err != nil { + break + } + if debug { + println("next block:", dec) + } + err = dec.decodeBuf(&d.history) + if err != nil || dec.Last { + break + } + if uint64(len(d.history.b)) > d.o.maxDecodedSize { + err = ErrDecoderSizeExceeded + break + } + if d.SingleSegment && uint64(len(d.history.b)) > d.o.maxDecodedSize { + println("runDecoder: single segment and", uint64(len(d.history.b)), ">", d.o.maxDecodedSize) + err = ErrFrameSizeExceeded + break + } + } + dst = d.history.b + if err == nil { + if d.HasCheckSum { + var n int + n, err = d.crc.Write(dst[crcStart:]) + if err == nil { + if n != len(dst)-crcStart { + err = io.ErrShortWrite + } else { + err = d.checkCRC() + } + } + } + } + d.history.b = saved + return dst, err +} diff --git a/vendor/github.com/klauspost/compress/zstd/frameenc.go b/vendor/github.com/klauspost/compress/zstd/frameenc.go new file mode 100644 index 000000000..4479cfe18 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/frameenc.go @@ -0,0 +1,115 @@ +// Copyright 2019+ Klaus Post. All rights reserved. +// License information can be found in the LICENSE file. +// Based on work by Yann Collet, released under BSD License. + +package zstd + +import ( + "fmt" + "io" + "math" + "math/bits" +) + +type frameHeader struct { + ContentSize uint64 + WindowSize uint32 + SingleSegment bool + Checksum bool + DictID uint32 // Not stored. +} + +const maxHeaderSize = 14 + +func (f frameHeader) appendTo(dst []byte) ([]byte, error) { + dst = append(dst, frameMagic...) + var fhd uint8 + if f.Checksum { + fhd |= 1 << 2 + } + if f.SingleSegment { + fhd |= 1 << 5 + } + var fcs uint8 + if f.ContentSize >= 256 { + fcs++ + } + if f.ContentSize >= 65536+256 { + fcs++ + } + if f.ContentSize >= 0xffffffff { + fcs++ + } + fhd |= fcs << 6 + + dst = append(dst, fhd) + if !f.SingleSegment { + const winLogMin = 10 + windowLog := (bits.Len32(f.WindowSize-1) - winLogMin) << 3 + dst = append(dst, uint8(windowLog)) + } + + switch fcs { + case 0: + if f.SingleSegment { + dst = append(dst, uint8(f.ContentSize)) + } + // Unless SingleSegment is set, framessizes < 256 are nto stored. + case 1: + f.ContentSize -= 256 + dst = append(dst, uint8(f.ContentSize), uint8(f.ContentSize>>8)) + case 2: + dst = append(dst, uint8(f.ContentSize), uint8(f.ContentSize>>8), uint8(f.ContentSize>>16), uint8(f.ContentSize>>24)) + case 3: + dst = append(dst, uint8(f.ContentSize), uint8(f.ContentSize>>8), uint8(f.ContentSize>>16), uint8(f.ContentSize>>24), + uint8(f.ContentSize>>32), uint8(f.ContentSize>>40), uint8(f.ContentSize>>48), uint8(f.ContentSize>>56)) + default: + panic("invalid fcs") + } + return dst, nil +} + +const skippableFrameHeader = 4 + 4 + +// calcSkippableFrame will return a total size to be added for written +// to be divisible by multiple. +// The value will always be > skippableFrameHeader. +// The function will panic if written < 0 or wantMultiple <= 0. +func calcSkippableFrame(written, wantMultiple int64) int { + if wantMultiple <= 0 { + panic("wantMultiple <= 0") + } + if written < 0 { + panic("written < 0") + } + leftOver := written % wantMultiple + if leftOver == 0 { + return 0 + } + toAdd := wantMultiple - leftOver + for toAdd < skippableFrameHeader { + toAdd += wantMultiple + } + return int(toAdd) +} + +// skippableFrame will add a skippable frame with a total size of bytes. +// total should be >= skippableFrameHeader and < math.MaxUint32. +func skippableFrame(dst []byte, total int, r io.Reader) ([]byte, error) { + if total == 0 { + return dst, nil + } + if total < skippableFrameHeader { + return dst, fmt.Errorf("requested skippable frame (%d) < 8", total) + } + if int64(total) > math.MaxUint32 { + return dst, fmt.Errorf("requested skippable frame (%d) > max uint32", total) + } + dst = append(dst, 0x50, 0x2a, 0x4d, 0x18) + f := uint32(total - skippableFrameHeader) + dst = append(dst, uint8(f), uint8(f>>8), uint8(f>>16), uint8(f>>24)) + start := len(dst) + dst = append(dst, make([]byte, f)...) + _, err := io.ReadFull(r, dst[start:]) + return dst, err +} diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder.go b/vendor/github.com/klauspost/compress/zstd/fse_decoder.go new file mode 100644 index 000000000..9efe34feb --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder.go @@ -0,0 +1,384 @@ +// Copyright 2019+ Klaus Post. All rights reserved. +// License information can be found in the LICENSE file. +// Based on work by Yann Collet, released under BSD License. + +package zstd + +import ( + "errors" + "fmt" +) + +const ( + tablelogAbsoluteMax = 9 +) + +const ( + /*!MEMORY_USAGE : + * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) + * Increasing memory usage improves compression ratio + * Reduced memory usage can improve speed, due to cache effect + * Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ + maxMemoryUsage = 11 + + maxTableLog = maxMemoryUsage - 2 + maxTablesize = 1 << maxTableLog + maxTableMask = (1 << maxTableLog) - 1 + minTablelog = 5 + maxSymbolValue = 255 +) + +// fseDecoder provides temporary storage for compression and decompression. +type fseDecoder struct { + dt [maxTablesize]decSymbol // Decompression table. + symbolLen uint16 // Length of active part of the symbol table. + actualTableLog uint8 // Selected tablelog. + maxBits uint8 // Maximum number of additional bits + + // used for table creation to avoid allocations. + stateTable [256]uint16 + norm [maxSymbolValue + 1]int16 + preDefined bool +} + +// tableStep returns the next table index. +func tableStep(tableSize uint32) uint32 { + return (tableSize >> 1) + (tableSize >> 3) + 3 +} + +// readNCount will read the symbol distribution so decoding tables can be constructed. +func (s *fseDecoder) readNCount(b *byteReader, maxSymbol uint16) error { + var ( + charnum uint16 + previous0 bool + ) + if b.remain() < 4 { + return errors.New("input too small") + } + bitStream := b.Uint32() + nbBits := uint((bitStream & 0xF) + minTablelog) // extract tableLog + if nbBits > tablelogAbsoluteMax { + println("Invalid tablelog:", nbBits) + return errors.New("tableLog too large") + } + bitStream >>= 4 + bitCount := uint(4) + + s.actualTableLog = uint8(nbBits) + remaining := int32((1 << nbBits) + 1) + threshold := int32(1 << nbBits) + gotTotal := int32(0) + nbBits++ + + for remaining > 1 && charnum <= maxSymbol { + if previous0 { + //println("prev0") + n0 := charnum + for (bitStream & 0xFFFF) == 0xFFFF { + //println("24 x 0") + n0 += 24 + if r := b.remain(); r > 5 { + b.advance(2) + bitStream = b.Uint32() >> bitCount + } else { + // end of bit stream + bitStream >>= 16 + bitCount += 16 + } + } + //printf("bitstream: %d, 0b%b", bitStream&3, bitStream) + for (bitStream & 3) == 3 { + n0 += 3 + bitStream >>= 2 + bitCount += 2 + } + n0 += uint16(bitStream & 3) + bitCount += 2 + + if n0 > maxSymbolValue { + return errors.New("maxSymbolValue too small") + } + //println("inserting ", n0-charnum, "zeroes from idx", charnum, "ending before", n0) + for charnum < n0 { + s.norm[uint8(charnum)] = 0 + charnum++ + } + + if r := b.remain(); r >= 7 || r+int(bitCount>>3) >= 4 { + b.advance(bitCount >> 3) + bitCount &= 7 + bitStream = b.Uint32() >> bitCount + } else { + bitStream >>= 2 + } + } + + max := (2*threshold - 1) - remaining + var count int32 + + if int32(bitStream)&(threshold-1) < max { + count = int32(bitStream) & (threshold - 1) + if debug && nbBits < 1 { + panic("nbBits underflow") + } + bitCount += nbBits - 1 + } else { + count = int32(bitStream) & (2*threshold - 1) + if count >= threshold { + count -= max + } + bitCount += nbBits + } + + // extra accuracy + count-- + if count < 0 { + // -1 means +1 + remaining += count + gotTotal -= count + } else { + remaining -= count + gotTotal += count + } + s.norm[charnum&0xff] = int16(count) + charnum++ + previous0 = count == 0 + for remaining < threshold { + nbBits-- + threshold >>= 1 + } + + //println("b.off:", b.off, "len:", len(b.b), "bc:", bitCount, "remain:", b.remain()) + if r := b.remain(); r >= 7 || r+int(bitCount>>3) >= 4 { + b.advance(bitCount >> 3) + bitCount &= 7 + } else { + bitCount -= (uint)(8 * (len(b.b) - 4 - b.off)) + b.off = len(b.b) - 4 + //println("b.off:", b.off, "len:", len(b.b), "bc:", bitCount, "iend", iend) + } + bitStream = b.Uint32() >> (bitCount & 31) + //printf("bitstream is now: 0b%b", bitStream) + } + s.symbolLen = charnum + if s.symbolLen <= 1 { + return fmt.Errorf("symbolLen (%d) too small", s.symbolLen) + } + if s.symbolLen > maxSymbolValue+1 { + return fmt.Errorf("symbolLen (%d) too big", s.symbolLen) + } + if remaining != 1 { + return fmt.Errorf("corruption detected (remaining %d != 1)", remaining) + } + if bitCount > 32 { + return fmt.Errorf("corruption detected (bitCount %d > 32)", bitCount) + } + if gotTotal != 1<> 3) + // println(s.norm[:s.symbolLen], s.symbolLen) + return s.buildDtable() +} + +// decSymbol contains information about a state entry, +// Including the state offset base, the output symbol and +// the number of bits to read for the low part of the destination state. +// Using a composite uint64 is faster than a struct with separate members. +type decSymbol uint64 + +func newDecSymbol(nbits, addBits uint8, newState uint16, baseline uint32) decSymbol { + return decSymbol(nbits) | (decSymbol(addBits) << 8) | (decSymbol(newState) << 16) | (decSymbol(baseline) << 32) +} + +func (d decSymbol) nbBits() uint8 { + return uint8(d) +} + +func (d decSymbol) addBits() uint8 { + return uint8(d >> 8) +} + +func (d decSymbol) newState() uint16 { + return uint16(d >> 16) +} + +func (d decSymbol) baseline() uint32 { + return uint32(d >> 32) +} + +func (d decSymbol) baselineInt() int { + return int(d >> 32) +} + +func (d *decSymbol) set(nbits, addBits uint8, newState uint16, baseline uint32) { + *d = decSymbol(nbits) | (decSymbol(addBits) << 8) | (decSymbol(newState) << 16) | (decSymbol(baseline) << 32) +} + +func (d *decSymbol) setNBits(nBits uint8) { + const mask = 0xffffffffffffff00 + *d = (*d & mask) | decSymbol(nBits) +} + +func (d *decSymbol) setAddBits(addBits uint8) { + const mask = 0xffffffffffff00ff + *d = (*d & mask) | (decSymbol(addBits) << 8) +} + +func (d *decSymbol) setNewState(state uint16) { + const mask = 0xffffffff0000ffff + *d = (*d & mask) | decSymbol(state)<<16 +} + +func (d *decSymbol) setBaseline(baseline uint32) { + const mask = 0xffffffff + *d = (*d & mask) | decSymbol(baseline)<<32 +} + +func (d *decSymbol) setExt(addBits uint8, baseline uint32) { + const mask = 0xffff00ff + *d = (*d & mask) | (decSymbol(addBits) << 8) | (decSymbol(baseline) << 32) +} + +// decSymbolValue returns the transformed decSymbol for the given symbol. +func decSymbolValue(symb uint8, t []baseOffset) (decSymbol, error) { + if int(symb) >= len(t) { + return 0, fmt.Errorf("rle symbol %d >= max %d", symb, len(t)) + } + lu := t[symb] + return newDecSymbol(0, lu.addBits, 0, lu.baseLine), nil +} + +// setRLE will set the decoder til RLE mode. +func (s *fseDecoder) setRLE(symbol decSymbol) { + s.actualTableLog = 0 + s.maxBits = symbol.addBits() + s.dt[0] = symbol +} + +// buildDtable will build the decoding table. +func (s *fseDecoder) buildDtable() error { + tableSize := uint32(1 << s.actualTableLog) + highThreshold := tableSize - 1 + symbolNext := s.stateTable[:256] + + // Init, lay down lowprob symbols + { + for i, v := range s.norm[:s.symbolLen] { + if v == -1 { + s.dt[highThreshold].setAddBits(uint8(i)) + highThreshold-- + symbolNext[i] = 1 + } else { + symbolNext[i] = uint16(v) + } + } + } + // Spread symbols + { + tableMask := tableSize - 1 + step := tableStep(tableSize) + position := uint32(0) + for ss, v := range s.norm[:s.symbolLen] { + for i := 0; i < int(v); i++ { + s.dt[position].setAddBits(uint8(ss)) + position = (position + step) & tableMask + for position > highThreshold { + // lowprob area + position = (position + step) & tableMask + } + } + } + if position != 0 { + // position must reach all cells once, otherwise normalizedCounter is incorrect + return errors.New("corrupted input (position != 0)") + } + } + + // Build Decoding table + { + tableSize := uint16(1 << s.actualTableLog) + for u, v := range s.dt[:tableSize] { + symbol := v.addBits() + nextState := symbolNext[symbol] + symbolNext[symbol] = nextState + 1 + nBits := s.actualTableLog - byte(highBits(uint32(nextState))) + s.dt[u&maxTableMask].setNBits(nBits) + newState := (nextState << nBits) - tableSize + if newState > tableSize { + return fmt.Errorf("newState (%d) outside table size (%d)", newState, tableSize) + } + if newState == uint16(u) && nBits == 0 { + // Seems weird that this is possible with nbits > 0. + return fmt.Errorf("newState (%d) == oldState (%d) and no bits", newState, u) + } + s.dt[u&maxTableMask].setNewState(newState) + } + } + return nil +} + +// transform will transform the decoder table into a table usable for +// decoding without having to apply the transformation while decoding. +// The state will contain the base value and the number of bits to read. +func (s *fseDecoder) transform(t []baseOffset) error { + tableSize := uint16(1 << s.actualTableLog) + s.maxBits = 0 + for i, v := range s.dt[:tableSize] { + add := v.addBits() + if int(add) >= len(t) { + return fmt.Errorf("invalid decoding table entry %d, symbol %d >= max (%d)", i, v.addBits(), len(t)) + } + lu := t[add] + if lu.addBits > s.maxBits { + s.maxBits = lu.addBits + } + v.setExt(lu.addBits, lu.baseLine) + s.dt[i] = v + } + return nil +} + +type fseState struct { + dt []decSymbol + state decSymbol +} + +// Initialize and decodeAsync first state and symbol. +func (s *fseState) init(br *bitReader, tableLog uint8, dt []decSymbol) { + s.dt = dt + br.fill() + s.state = dt[br.getBits(tableLog)] +} + +// next returns the current symbol and sets the next state. +// At least tablelog bits must be available in the bit reader. +func (s *fseState) next(br *bitReader) { + lowBits := uint16(br.getBits(s.state.nbBits())) + s.state = s.dt[s.state.newState()+lowBits] +} + +// finished returns true if all bits have been read from the bitstream +// and the next state would require reading bits from the input. +func (s *fseState) finished(br *bitReader) bool { + return br.finished() && s.state.nbBits() > 0 +} + +// final returns the current state symbol without decoding the next. +func (s *fseState) final() (int, uint8) { + return s.state.baselineInt(), s.state.addBits() +} + +// final returns the current state symbol without decoding the next. +func (s decSymbol) final() (int, uint8) { + return s.baselineInt(), s.addBits() +} + +// nextFast returns the next symbol and sets the next state. +// This can only be used if no symbols are 0 bits. +// At least tablelog bits must be available in the bit reader. +func (s *fseState) nextFast(br *bitReader) (uint32, uint8) { + lowBits := uint16(br.getBitsFast(s.state.nbBits())) + s.state = s.dt[s.state.newState()+lowBits] + return s.state.baseline(), s.state.addBits() +} diff --git a/vendor/github.com/klauspost/compress/zstd/fse_encoder.go b/vendor/github.com/klauspost/compress/zstd/fse_encoder.go new file mode 100644 index 000000000..619836f52 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/fse_encoder.go @@ -0,0 +1,726 @@ +// Copyright 2019+ Klaus Post. All rights reserved. +// License information can be found in the LICENSE file. +// Based on work by Yann Collet, released under BSD License. + +package zstd + +import ( + "errors" + "fmt" + "math" +) + +const ( + // For encoding we only support up to + maxEncTableLog = 8 + maxEncTablesize = 1 << maxTableLog + maxEncTableMask = (1 << maxTableLog) - 1 + minEncTablelog = 5 + maxEncSymbolValue = maxMatchLengthSymbol +) + +// Scratch provides temporary storage for compression and decompression. +type fseEncoder struct { + symbolLen uint16 // Length of active part of the symbol table. + actualTableLog uint8 // Selected tablelog. + ct cTable // Compression tables. + maxCount int // count of the most probable symbol + zeroBits bool // no bits has prob > 50%. + clearCount bool // clear count + useRLE bool // This encoder is for RLE + preDefined bool // This encoder is predefined. + reUsed bool // Set to know when the encoder has been reused. + rleVal uint8 // RLE Symbol + maxBits uint8 // Maximum output bits after transform. + + // TODO: Technically zstd should be fine with 64 bytes. + count [256]uint32 + norm [256]int16 +} + +// cTable contains tables used for compression. +type cTable struct { + tableSymbol []byte + stateTable []uint16 + symbolTT []symbolTransform +} + +// symbolTransform contains the state transform for a symbol. +type symbolTransform struct { + deltaNbBits uint32 + deltaFindState int16 + outBits uint8 +} + +// String prints values as a human readable string. +func (s symbolTransform) String() string { + return fmt.Sprintf("{deltabits: %08x, findstate:%d outbits:%d}", s.deltaNbBits, s.deltaFindState, s.outBits) +} + +// Histogram allows to populate the histogram and skip that step in the compression, +// It otherwise allows to inspect the histogram when compression is done. +// To indicate that you have populated the histogram call HistogramFinished +// with the value of the highest populated symbol, as well as the number of entries +// in the most populated entry. These are accepted at face value. +// The returned slice will always be length 256. +func (s *fseEncoder) Histogram() []uint32 { + return s.count[:] +} + +// HistogramFinished can be called to indicate that the histogram has been populated. +// maxSymbol is the index of the highest set symbol of the next data segment. +// maxCount is the number of entries in the most populated entry. +// These are accepted at face value. +func (s *fseEncoder) HistogramFinished(maxSymbol uint8, maxCount int) { + s.maxCount = maxCount + s.symbolLen = uint16(maxSymbol) + 1 + s.clearCount = maxCount != 0 +} + +// prepare will prepare and allocate scratch tables used for both compression and decompression. +func (s *fseEncoder) prepare() (*fseEncoder, error) { + if s == nil { + s = &fseEncoder{} + } + s.useRLE = false + if s.clearCount && s.maxCount == 0 { + for i := range s.count { + s.count[i] = 0 + } + s.clearCount = false + } + return s, nil +} + +// allocCtable will allocate tables needed for compression. +// If existing tables a re big enough, they are simply re-used. +func (s *fseEncoder) allocCtable() { + tableSize := 1 << s.actualTableLog + // get tableSymbol that is big enough. + if cap(s.ct.tableSymbol) < int(tableSize) { + s.ct.tableSymbol = make([]byte, tableSize) + } + s.ct.tableSymbol = s.ct.tableSymbol[:tableSize] + + ctSize := tableSize + if cap(s.ct.stateTable) < ctSize { + s.ct.stateTable = make([]uint16, ctSize) + } + s.ct.stateTable = s.ct.stateTable[:ctSize] + + if cap(s.ct.symbolTT) < 256 { + s.ct.symbolTT = make([]symbolTransform, 256) + } + s.ct.symbolTT = s.ct.symbolTT[:256] +} + +// buildCTable will populate the compression table so it is ready to be used. +func (s *fseEncoder) buildCTable() error { + tableSize := uint32(1 << s.actualTableLog) + highThreshold := tableSize - 1 + var cumul [256]int16 + + s.allocCtable() + tableSymbol := s.ct.tableSymbol[:tableSize] + // symbol start positions + { + cumul[0] = 0 + for ui, v := range s.norm[:s.symbolLen-1] { + u := byte(ui) // one less than reference + if v == -1 { + // Low proba symbol + cumul[u+1] = cumul[u] + 1 + tableSymbol[highThreshold] = u + highThreshold-- + } else { + cumul[u+1] = cumul[u] + v + } + } + // Encode last symbol separately to avoid overflowing u + u := int(s.symbolLen - 1) + v := s.norm[s.symbolLen-1] + if v == -1 { + // Low proba symbol + cumul[u+1] = cumul[u] + 1 + tableSymbol[highThreshold] = byte(u) + highThreshold-- + } else { + cumul[u+1] = cumul[u] + v + } + if uint32(cumul[s.symbolLen]) != tableSize { + return fmt.Errorf("internal error: expected cumul[s.symbolLen] (%d) == tableSize (%d)", cumul[s.symbolLen], tableSize) + } + cumul[s.symbolLen] = int16(tableSize) + 1 + } + // Spread symbols + s.zeroBits = false + { + step := tableStep(tableSize) + tableMask := tableSize - 1 + var position uint32 + // if any symbol > largeLimit, we may have 0 bits output. + largeLimit := int16(1 << (s.actualTableLog - 1)) + for ui, v := range s.norm[:s.symbolLen] { + symbol := byte(ui) + if v > largeLimit { + s.zeroBits = true + } + for nbOccurrences := int16(0); nbOccurrences < v; nbOccurrences++ { + tableSymbol[position] = symbol + position = (position + step) & tableMask + for position > highThreshold { + position = (position + step) & tableMask + } /* Low proba area */ + } + } + + // Check if we have gone through all positions + if position != 0 { + return errors.New("position!=0") + } + } + + // Build table + table := s.ct.stateTable + { + tsi := int(tableSize) + for u, v := range tableSymbol { + // TableU16 : sorted by symbol order; gives next state value + table[cumul[v]] = uint16(tsi + u) + cumul[v]++ + } + } + + // Build Symbol Transformation Table + { + total := int16(0) + symbolTT := s.ct.symbolTT[:s.symbolLen] + tableLog := s.actualTableLog + tl := (uint32(tableLog) << 16) - (1 << tableLog) + for i, v := range s.norm[:s.symbolLen] { + switch v { + case 0: + case -1, 1: + symbolTT[i].deltaNbBits = tl + symbolTT[i].deltaFindState = int16(total - 1) + total++ + default: + maxBitsOut := uint32(tableLog) - highBit(uint32(v-1)) + minStatePlus := uint32(v) << maxBitsOut + symbolTT[i].deltaNbBits = (maxBitsOut << 16) - minStatePlus + symbolTT[i].deltaFindState = int16(total - v) + total += v + } + } + if total != int16(tableSize) { + return fmt.Errorf("total mismatch %d (got) != %d (want)", total, tableSize) + } + } + return nil +} + +var rtbTable = [...]uint32{0, 473195, 504333, 520860, 550000, 700000, 750000, 830000} + +func (s *fseEncoder) setRLE(val byte) { + s.allocCtable() + s.actualTableLog = 0 + s.ct.stateTable = s.ct.stateTable[:1] + s.ct.symbolTT[val] = symbolTransform{ + deltaFindState: 0, + deltaNbBits: 0, + } + if debug { + println("setRLE: val", val, "symbolTT", s.ct.symbolTT[val]) + } + s.rleVal = val + s.useRLE = true +} + +// setBits will set output bits for the transform. +// if nil is provided, the number of bits is equal to the index. +func (s *fseEncoder) setBits(transform []byte) { + if s.reUsed || s.preDefined { + return + } + if s.useRLE { + if transform == nil { + s.ct.symbolTT[s.rleVal].outBits = s.rleVal + s.maxBits = s.rleVal + return + } + s.maxBits = transform[s.rleVal] + s.ct.symbolTT[s.rleVal].outBits = s.maxBits + return + } + if transform == nil { + for i := range s.ct.symbolTT[:s.symbolLen] { + s.ct.symbolTT[i].outBits = uint8(i) + } + s.maxBits = uint8(s.symbolLen - 1) + return + } + s.maxBits = 0 + for i, v := range transform[:s.symbolLen] { + s.ct.symbolTT[i].outBits = v + if v > s.maxBits { + // We could assume bits always going up, but we play safe. + s.maxBits = v + } + } +} + +// normalizeCount will normalize the count of the symbols so +// the total is equal to the table size. +// If successful, compression tables will also be made ready. +func (s *fseEncoder) normalizeCount(length int) error { + if s.reUsed { + return nil + } + s.optimalTableLog(length) + var ( + tableLog = s.actualTableLog + scale = 62 - uint64(tableLog) + step = (1 << 62) / uint64(length) + vStep = uint64(1) << (scale - 20) + stillToDistribute = int16(1 << tableLog) + largest int + largestP int16 + lowThreshold = (uint32)(length >> tableLog) + ) + if s.maxCount == length { + s.useRLE = true + return nil + } + s.useRLE = false + for i, cnt := range s.count[:s.symbolLen] { + // already handled + // if (count[s] == s.length) return 0; /* rle special case */ + + if cnt == 0 { + s.norm[i] = 0 + continue + } + if cnt <= lowThreshold { + s.norm[i] = -1 + stillToDistribute-- + } else { + proba := (int16)((uint64(cnt) * step) >> scale) + if proba < 8 { + restToBeat := vStep * uint64(rtbTable[proba]) + v := uint64(cnt)*step - (uint64(proba) << scale) + if v > restToBeat { + proba++ + } + } + if proba > largestP { + largestP = proba + largest = i + } + s.norm[i] = proba + stillToDistribute -= proba + } + } + + if -stillToDistribute >= (s.norm[largest] >> 1) { + // corner case, need another normalization method + err := s.normalizeCount2(length) + if err != nil { + return err + } + if debug { + err = s.validateNorm() + if err != nil { + return err + } + } + return s.buildCTable() + } + s.norm[largest] += stillToDistribute + if debug { + err := s.validateNorm() + if err != nil { + return err + } + } + return s.buildCTable() +} + +// Secondary normalization method. +// To be used when primary method fails. +func (s *fseEncoder) normalizeCount2(length int) error { + const notYetAssigned = -2 + var ( + distributed uint32 + total = uint32(length) + tableLog = s.actualTableLog + lowThreshold = uint32(total >> tableLog) + lowOne = uint32((total * 3) >> (tableLog + 1)) + ) + for i, cnt := range s.count[:s.symbolLen] { + if cnt == 0 { + s.norm[i] = 0 + continue + } + if cnt <= lowThreshold { + s.norm[i] = -1 + distributed++ + total -= cnt + continue + } + if cnt <= lowOne { + s.norm[i] = 1 + distributed++ + total -= cnt + continue + } + s.norm[i] = notYetAssigned + } + toDistribute := (1 << tableLog) - distributed + + if (total / toDistribute) > lowOne { + // risk of rounding to zero + lowOne = uint32((total * 3) / (toDistribute * 2)) + for i, cnt := range s.count[:s.symbolLen] { + if (s.norm[i] == notYetAssigned) && (cnt <= lowOne) { + s.norm[i] = 1 + distributed++ + total -= cnt + continue + } + } + toDistribute = (1 << tableLog) - distributed + } + if distributed == uint32(s.symbolLen)+1 { + // all values are pretty poor; + // probably incompressible data (should have already been detected); + // find max, then give all remaining points to max + var maxV int + var maxC uint32 + for i, cnt := range s.count[:s.symbolLen] { + if cnt > maxC { + maxV = i + maxC = cnt + } + } + s.norm[maxV] += int16(toDistribute) + return nil + } + + if total == 0 { + // all of the symbols were low enough for the lowOne or lowThreshold + for i := uint32(0); toDistribute > 0; i = (i + 1) % (uint32(s.symbolLen)) { + if s.norm[i] > 0 { + toDistribute-- + s.norm[i]++ + } + } + return nil + } + + var ( + vStepLog = 62 - uint64(tableLog) + mid = uint64((1 << (vStepLog - 1)) - 1) + rStep = (((1 << vStepLog) * uint64(toDistribute)) + mid) / uint64(total) // scale on remaining + tmpTotal = mid + ) + for i, cnt := range s.count[:s.symbolLen] { + if s.norm[i] == notYetAssigned { + var ( + end = tmpTotal + uint64(cnt)*rStep + sStart = uint32(tmpTotal >> vStepLog) + sEnd = uint32(end >> vStepLog) + weight = sEnd - sStart + ) + if weight < 1 { + return errors.New("weight < 1") + } + s.norm[i] = int16(weight) + tmpTotal = end + } + } + return nil +} + +// optimalTableLog calculates and sets the optimal tableLog in s.actualTableLog +func (s *fseEncoder) optimalTableLog(length int) { + tableLog := uint8(maxEncTableLog) + minBitsSrc := highBit(uint32(length)) + 1 + minBitsSymbols := highBit(uint32(s.symbolLen-1)) + 2 + minBits := uint8(minBitsSymbols) + if minBitsSrc < minBitsSymbols { + minBits = uint8(minBitsSrc) + } + + maxBitsSrc := uint8(highBit(uint32(length-1))) - 2 + if maxBitsSrc < tableLog { + // Accuracy can be reduced + tableLog = maxBitsSrc + } + if minBits > tableLog { + tableLog = minBits + } + // Need a minimum to safely represent all symbol values + if tableLog < minEncTablelog { + tableLog = minEncTablelog + } + if tableLog > maxEncTableLog { + tableLog = maxEncTableLog + } + s.actualTableLog = tableLog +} + +// validateNorm validates the normalized histogram table. +func (s *fseEncoder) validateNorm() (err error) { + var total int + for _, v := range s.norm[:s.symbolLen] { + if v >= 0 { + total += int(v) + } else { + total -= int(v) + } + } + defer func() { + if err == nil { + return + } + fmt.Printf("selected TableLog: %d, Symbol length: %d\n", s.actualTableLog, s.symbolLen) + for i, v := range s.norm[:s.symbolLen] { + fmt.Printf("%3d: %5d -> %4d \n", i, s.count[i], v) + } + }() + if total != (1 << s.actualTableLog) { + return fmt.Errorf("warning: Total == %d != %d", total, 1<> 3) + 3 + 2 + + // Write Table Size + bitStream = uint32(tableLog - minEncTablelog) + bitCount = uint(4) + remaining = int16(tableSize + 1) /* +1 for extra accuracy */ + threshold = int16(tableSize) + nbBits = uint(tableLog + 1) + outP = len(out) + ) + if cap(out) < outP+maxHeaderSize { + out = append(out, make([]byte, maxHeaderSize*3)...) + out = out[:len(out)-maxHeaderSize*3] + } + out = out[:outP+maxHeaderSize] + + // stops at 1 + for remaining > 1 { + if previous0 { + start := charnum + for s.norm[charnum] == 0 { + charnum++ + } + for charnum >= start+24 { + start += 24 + bitStream += uint32(0xFFFF) << bitCount + out[outP] = byte(bitStream) + out[outP+1] = byte(bitStream >> 8) + outP += 2 + bitStream >>= 16 + } + for charnum >= start+3 { + start += 3 + bitStream += 3 << bitCount + bitCount += 2 + } + bitStream += uint32(charnum-start) << bitCount + bitCount += 2 + if bitCount > 16 { + out[outP] = byte(bitStream) + out[outP+1] = byte(bitStream >> 8) + outP += 2 + bitStream >>= 16 + bitCount -= 16 + } + } + + count := s.norm[charnum] + charnum++ + max := (2*threshold - 1) - remaining + if count < 0 { + remaining += count + } else { + remaining -= count + } + count++ // +1 for extra accuracy + if count >= threshold { + count += max // [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ + } + bitStream += uint32(count) << bitCount + bitCount += nbBits + if count < max { + bitCount-- + } + + previous0 = count == 1 + if remaining < 1 { + return nil, errors.New("internal error: remaining < 1") + } + for remaining < threshold { + nbBits-- + threshold >>= 1 + } + + if bitCount > 16 { + out[outP] = byte(bitStream) + out[outP+1] = byte(bitStream >> 8) + outP += 2 + bitStream >>= 16 + bitCount -= 16 + } + } + + if outP+2 > len(out) { + return nil, fmt.Errorf("internal error: %d > %d, maxheader: %d, sl: %d, tl: %d, normcount: %v", outP+2, len(out), maxHeaderSize, s.symbolLen, int(tableLog), s.norm[:s.symbolLen]) + } + out[outP] = byte(bitStream) + out[outP+1] = byte(bitStream >> 8) + outP += int((bitCount + 7) / 8) + + if charnum > s.symbolLen { + return nil, errors.New("internal error: charnum > s.symbolLen") + } + return out[:outP], nil +} + +// Approximate symbol cost, as fractional value, using fixed-point format (accuracyLog fractional bits) +// note 1 : assume symbolValue is valid (<= maxSymbolValue) +// note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits * +func (s *fseEncoder) bitCost(symbolValue uint8, accuracyLog uint32) uint32 { + minNbBits := s.ct.symbolTT[symbolValue].deltaNbBits >> 16 + threshold := (minNbBits + 1) << 16 + if debug { + if !(s.actualTableLog < 16) { + panic("!s.actualTableLog < 16") + } + // ensure enough room for renormalization double shift + if !(uint8(accuracyLog) < 31-s.actualTableLog) { + panic("!uint8(accuracyLog) < 31-s.actualTableLog") + } + } + tableSize := uint32(1) << s.actualTableLog + deltaFromThreshold := threshold - (s.ct.symbolTT[symbolValue].deltaNbBits + tableSize) + // linear interpolation (very approximate) + normalizedDeltaFromThreshold := (deltaFromThreshold << accuracyLog) >> s.actualTableLog + bitMultiplier := uint32(1) << accuracyLog + if debug { + if s.ct.symbolTT[symbolValue].deltaNbBits+tableSize > threshold { + panic("s.ct.symbolTT[symbolValue].deltaNbBits+tableSize > threshold") + } + if normalizedDeltaFromThreshold > bitMultiplier { + panic("normalizedDeltaFromThreshold > bitMultiplier") + } + } + return (minNbBits+1)*bitMultiplier - normalizedDeltaFromThreshold +} + +// Returns the cost in bits of encoding the distribution in count using ctable. +// Histogram should only be up to the last non-zero symbol. +// Returns an -1 if ctable cannot represent all the symbols in count. +func (s *fseEncoder) approxSize(hist []uint32) uint32 { + if int(s.symbolLen) < len(hist) { + // More symbols than we have. + return math.MaxUint32 + } + if s.useRLE { + // We will never reuse RLE encoders. + return math.MaxUint32 + } + const kAccuracyLog = 8 + badCost := (uint32(s.actualTableLog) + 1) << kAccuracyLog + var cost uint32 + for i, v := range hist { + if v == 0 { + continue + } + if s.norm[i] == 0 { + return math.MaxUint32 + } + bitCost := s.bitCost(uint8(i), kAccuracyLog) + if bitCost > badCost { + return math.MaxUint32 + } + cost += v * bitCost + } + return cost >> kAccuracyLog +} + +// maxHeaderSize returns the maximum header size in bits. +// This is not exact size, but we want a penalty for new tables anyway. +func (s *fseEncoder) maxHeaderSize() uint32 { + if s.preDefined { + return 0 + } + if s.useRLE { + return 8 + } + return (((uint32(s.symbolLen) * uint32(s.actualTableLog)) >> 3) + 3) * 8 +} + +// cState contains the compression state of a stream. +type cState struct { + bw *bitWriter + stateTable []uint16 + state uint16 +} + +// init will initialize the compression state to the first symbol of the stream. +func (c *cState) init(bw *bitWriter, ct *cTable, first symbolTransform) { + c.bw = bw + c.stateTable = ct.stateTable + if len(c.stateTable) == 1 { + // RLE + c.stateTable[0] = uint16(0) + c.state = 0 + return + } + nbBitsOut := (first.deltaNbBits + (1 << 15)) >> 16 + im := int32((nbBitsOut << 16) - first.deltaNbBits) + lu := (im >> nbBitsOut) + int32(first.deltaFindState) + c.state = c.stateTable[lu] + return +} + +// encode the output symbol provided and write it to the bitstream. +func (c *cState) encode(symbolTT symbolTransform) { + nbBitsOut := (uint32(c.state) + symbolTT.deltaNbBits) >> 16 + dstState := int32(c.state>>(nbBitsOut&15)) + int32(symbolTT.deltaFindState) + c.bw.addBits16NC(c.state, uint8(nbBitsOut)) + c.state = c.stateTable[dstState] +} + +// flush will write the tablelog to the output and flush the remaining full bytes. +func (c *cState) flush(tableLog uint8) { + c.bw.flush32() + c.bw.addBits16NC(c.state, tableLog) +} diff --git a/vendor/github.com/klauspost/compress/zstd/fse_predefined.go b/vendor/github.com/klauspost/compress/zstd/fse_predefined.go new file mode 100644 index 000000000..6c17dc17f --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/fse_predefined.go @@ -0,0 +1,158 @@ +// Copyright 2019+ Klaus Post. All rights reserved. +// License information can be found in the LICENSE file. +// Based on work by Yann Collet, released under BSD License. + +package zstd + +import ( + "fmt" + "math" + "sync" +) + +var ( + // fsePredef are the predefined fse tables as defined here: + // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#default-distributions + // These values are already transformed. + fsePredef [3]fseDecoder + + // fsePredefEnc are the predefined encoder based on fse tables as defined here: + // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#default-distributions + // These values are already transformed. + fsePredefEnc [3]fseEncoder + + // symbolTableX contain the transformations needed for each type as defined in + // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#the-codes-for-literals-lengths-match-lengths-and-offsets + symbolTableX [3][]baseOffset + + // maxTableSymbol is the biggest supported symbol for each table type + // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#the-codes-for-literals-lengths-match-lengths-and-offsets + maxTableSymbol = [3]uint8{tableLiteralLengths: maxLiteralLengthSymbol, tableOffsets: maxOffsetLengthSymbol, tableMatchLengths: maxMatchLengthSymbol} + + // bitTables is the bits table for each table. + bitTables = [3][]byte{tableLiteralLengths: llBitsTable[:], tableOffsets: nil, tableMatchLengths: mlBitsTable[:]} +) + +type tableIndex uint8 + +const ( + // indexes for fsePredef and symbolTableX + tableLiteralLengths tableIndex = 0 + tableOffsets tableIndex = 1 + tableMatchLengths tableIndex = 2 + + maxLiteralLengthSymbol = 35 + maxOffsetLengthSymbol = 30 + maxMatchLengthSymbol = 52 +) + +// baseOffset is used for calculating transformations. +type baseOffset struct { + baseLine uint32 + addBits uint8 +} + +// fillBase will precalculate base offsets with the given bit distributions. +func fillBase(dst []baseOffset, base uint32, bits ...uint8) { + if len(bits) != len(dst) { + panic(fmt.Sprintf("len(dst) (%d) != len(bits) (%d)", len(dst), len(bits))) + } + for i, bit := range bits { + if base > math.MaxInt32 { + panic(fmt.Sprintf("invalid decoding table, base overflows int32")) + } + + dst[i] = baseOffset{ + baseLine: base, + addBits: bit, + } + base += 1 << bit + } +} + +var predef sync.Once + +func initPredefined() { + predef.Do(func() { + // Literals length codes + tmp := make([]baseOffset, 36) + for i := range tmp[:16] { + tmp[i] = baseOffset{ + baseLine: uint32(i), + addBits: 0, + } + } + fillBase(tmp[16:], 16, 1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16) + symbolTableX[tableLiteralLengths] = tmp + + // Match length codes + tmp = make([]baseOffset, 53) + for i := range tmp[:32] { + tmp[i] = baseOffset{ + // The transformation adds the 3 length. + baseLine: uint32(i) + 3, + addBits: 0, + } + } + fillBase(tmp[32:], 35, 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16) + symbolTableX[tableMatchLengths] = tmp + + // Offset codes + tmp = make([]baseOffset, maxOffsetBits+1) + tmp[1] = baseOffset{ + baseLine: 1, + addBits: 1, + } + fillBase(tmp[2:], 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30) + symbolTableX[tableOffsets] = tmp + + // Fill predefined tables and transform them. + // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#default-distributions + for i := range fsePredef[:] { + f := &fsePredef[i] + switch tableIndex(i) { + case tableLiteralLengths: + // https://github.com/facebook/zstd/blob/ededcfca57366461021c922720878c81a5854a0a/lib/decompress/zstd_decompress_block.c#L243 + f.actualTableLog = 6 + copy(f.norm[:], []int16{4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1, + -1, -1, -1, -1}) + f.symbolLen = 36 + case tableOffsets: + // https://github.com/facebook/zstd/blob/ededcfca57366461021c922720878c81a5854a0a/lib/decompress/zstd_decompress_block.c#L281 + f.actualTableLog = 5 + copy(f.norm[:], []int16{ + 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1}) + f.symbolLen = 29 + case tableMatchLengths: + //https://github.com/facebook/zstd/blob/ededcfca57366461021c922720878c81a5854a0a/lib/decompress/zstd_decompress_block.c#L304 + f.actualTableLog = 6 + copy(f.norm[:], []int16{ + 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, + -1, -1, -1, -1, -1}) + f.symbolLen = 53 + } + if err := f.buildDtable(); err != nil { + panic(fmt.Errorf("building table %v: %v", tableIndex(i), err)) + } + if err := f.transform(symbolTableX[i]); err != nil { + panic(fmt.Errorf("building table %v: %v", tableIndex(i), err)) + } + f.preDefined = true + + // Create encoder as well + enc := &fsePredefEnc[i] + copy(enc.norm[:], f.norm[:]) + enc.symbolLen = f.symbolLen + enc.actualTableLog = f.actualTableLog + if err := enc.buildCTable(); err != nil { + panic(fmt.Errorf("building encoding table %v: %v", tableIndex(i), err)) + } + enc.setBits(bitTables[i]) + enc.preDefined = true + } + }) +} diff --git a/vendor/github.com/klauspost/compress/zstd/hash.go b/vendor/github.com/klauspost/compress/zstd/hash.go new file mode 100644 index 000000000..4a752067f --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/hash.go @@ -0,0 +1,77 @@ +// Copyright 2019+ Klaus Post. All rights reserved. +// License information can be found in the LICENSE file. +// Based on work by Yann Collet, released under BSD License. + +package zstd + +const ( + prime3bytes = 506832829 + prime4bytes = 2654435761 + prime5bytes = 889523592379 + prime6bytes = 227718039650203 + prime7bytes = 58295818150454627 + prime8bytes = 0xcf1bbcdcb7a56463 +) + +// hashLen returns a hash of the lowest l bytes of u for a size size of h bytes. +// l must be >=4 and <=8. Any other value will return hash for 4 bytes. +// h should always be <32. +// Preferably h and l should be a constant. +// FIXME: This does NOT get resolved, if 'mls' is constant, +// so this cannot be used. +func hashLen(u uint64, hashLog, mls uint8) uint32 { + switch mls { + case 5: + return hash5(u, hashLog) + case 6: + return hash6(u, hashLog) + case 7: + return hash7(u, hashLog) + case 8: + return hash8(u, hashLog) + default: + return hash4x64(u, hashLog) + } +} + +// hash3 returns the hash of the lower 3 bytes of u to fit in a hash table with h bits. +// Preferably h should be a constant and should always be <32. +func hash3(u uint32, h uint8) uint32 { + return ((u << (32 - 24)) * prime3bytes) >> ((32 - h) & 31) +} + +// hash4 returns the hash of u to fit in a hash table with h bits. +// Preferably h should be a constant and should always be <32. +func hash4(u uint32, h uint8) uint32 { + return (u * prime4bytes) >> ((32 - h) & 31) +} + +// hash4x64 returns the hash of the lowest 4 bytes of u to fit in a hash table with h bits. +// Preferably h should be a constant and should always be <32. +func hash4x64(u uint64, h uint8) uint32 { + return (uint32(u) * prime4bytes) >> ((32 - h) & 31) +} + +// hash5 returns the hash of the lowest 5 bytes of u to fit in a hash table with h bits. +// Preferably h should be a constant and should always be <64. +func hash5(u uint64, h uint8) uint32 { + return uint32(((u << (64 - 40)) * prime5bytes) >> ((64 - h) & 63)) +} + +// hash6 returns the hash of the lowest 6 bytes of u to fit in a hash table with h bits. +// Preferably h should be a constant and should always be <64. +func hash6(u uint64, h uint8) uint32 { + return uint32(((u << (64 - 48)) * prime6bytes) >> ((64 - h) & 63)) +} + +// hash7 returns the hash of the lowest 7 bytes of u to fit in a hash table with h bits. +// Preferably h should be a constant and should always be <64. +func hash7(u uint64, h uint8) uint32 { + return uint32(((u << (64 - 56)) * prime7bytes) >> ((64 - h) & 63)) +} + +// hash8 returns the hash of u to fit in a hash table with h bits. +// Preferably h should be a constant and should always be <64. +func hash8(u uint64, h uint8) uint32 { + return uint32((u * prime8bytes) >> ((64 - h) & 63)) +} diff --git a/vendor/github.com/klauspost/compress/zstd/history.go b/vendor/github.com/klauspost/compress/zstd/history.go new file mode 100644 index 000000000..e8c419bd5 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/history.go @@ -0,0 +1,73 @@ +// Copyright 2019+ Klaus Post. All rights reserved. +// License information can be found in the LICENSE file. +// Based on work by Yann Collet, released under BSD License. + +package zstd + +import ( + "github.com/klauspost/compress/huff0" +) + +// history contains the information transferred between blocks. +type history struct { + b []byte + huffTree *huff0.Scratch + recentOffsets [3]int + decoders sequenceDecs + windowSize int + maxSize int + error bool +} + +// reset will reset the history to initial state of a frame. +// The history must already have been initialized to the desired size. +func (h *history) reset() { + h.b = h.b[:0] + h.error = false + h.recentOffsets = [3]int{1, 4, 8} + if f := h.decoders.litLengths.fse; f != nil && !f.preDefined { + fseDecoderPool.Put(f) + } + if f := h.decoders.offsets.fse; f != nil && !f.preDefined { + fseDecoderPool.Put(f) + } + if f := h.decoders.matchLengths.fse; f != nil && !f.preDefined { + fseDecoderPool.Put(f) + } + h.decoders = sequenceDecs{} + if h.huffTree != nil { + huffDecoderPool.Put(h.huffTree) + } + h.huffTree = nil + //printf("history created: %+v (l: %d, c: %d)", *h, len(h.b), cap(h.b)) +} + +// append bytes to history. +// This function will make sure there is space for it, +// if the buffer has been allocated with enough extra space. +func (h *history) append(b []byte) { + if len(b) >= h.windowSize { + // Discard all history by simply overwriting + h.b = h.b[:h.windowSize] + copy(h.b, b[len(b)-h.windowSize:]) + return + } + + // If there is space, append it. + if len(b) < cap(h.b)-len(h.b) { + h.b = append(h.b, b...) + return + } + + // Move data down so we only have window size left. + // We know we have less than window size in b at this point. + discard := len(b) + len(h.b) - h.windowSize + copy(h.b, h.b[discard:]) + h.b = h.b[:h.windowSize] + copy(h.b[h.windowSize-len(b):], b) +} + +// append bytes to history without ever discarding anything. +func (h *history) appendKeep(b []byte) { + h.b = append(h.b, b...) +} diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/LICENSE.txt b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/LICENSE.txt new file mode 100644 index 000000000..24b53065f --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/LICENSE.txt @@ -0,0 +1,22 @@ +Copyright (c) 2016 Caleb Spare + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/README.md b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/README.md new file mode 100644 index 000000000..69aa3bb58 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/README.md @@ -0,0 +1,58 @@ +# xxhash + +VENDORED: Go to [github.com/cespare/xxhash](https://github.com/cespare/xxhash) for original package. + + +[![GoDoc](https://godoc.org/github.com/cespare/xxhash?status.svg)](https://godoc.org/github.com/cespare/xxhash) +[![Build Status](https://travis-ci.org/cespare/xxhash.svg?branch=master)](https://travis-ci.org/cespare/xxhash) + +xxhash is a Go implementation of the 64-bit +[xxHash](http://cyan4973.github.io/xxHash/) algorithm, XXH64. This is a +high-quality hashing algorithm that is much faster than anything in the Go +standard library. + +This package provides a straightforward API: + +``` +func Sum64(b []byte) uint64 +func Sum64String(s string) uint64 +type Digest struct{ ... } + func New() *Digest +``` + +The `Digest` type implements hash.Hash64. Its key methods are: + +``` +func (*Digest) Write([]byte) (int, error) +func (*Digest) WriteString(string) (int, error) +func (*Digest) Sum64() uint64 +``` + +This implementation provides a fast pure-Go implementation and an even faster +assembly implementation for amd64. + +## Benchmarks + +Here are some quick benchmarks comparing the pure-Go and assembly +implementations of Sum64. + +| input size | purego | asm | +| --- | --- | --- | +| 5 B | 979.66 MB/s | 1291.17 MB/s | +| 100 B | 7475.26 MB/s | 7973.40 MB/s | +| 4 KB | 17573.46 MB/s | 17602.65 MB/s | +| 10 MB | 17131.46 MB/s | 17142.16 MB/s | + +These numbers were generated on Ubuntu 18.04 with an Intel i7-8700K CPU using +the following commands under Go 1.11.2: + +``` +$ go test -tags purego -benchtime 10s -bench '/xxhash,direct,bytes' +$ go test -benchtime 10s -bench '/xxhash,direct,bytes' +``` + +## Projects using this package + +- [InfluxDB](https://github.com/influxdata/influxdb) +- [Prometheus](https://github.com/prometheus/prometheus) +- [FreeCache](https://github.com/coocood/freecache) diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go new file mode 100644 index 000000000..426b9cac7 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go @@ -0,0 +1,238 @@ +// Package xxhash implements the 64-bit variant of xxHash (XXH64) as described +// at http://cyan4973.github.io/xxHash/. +// THIS IS VENDORED: Go to github.com/cespare/xxhash for original package. + +package xxhash + +import ( + "encoding/binary" + "errors" + "math/bits" +) + +const ( + prime1 uint64 = 11400714785074694791 + prime2 uint64 = 14029467366897019727 + prime3 uint64 = 1609587929392839161 + prime4 uint64 = 9650029242287828579 + prime5 uint64 = 2870177450012600261 +) + +// NOTE(caleb): I'm using both consts and vars of the primes. Using consts where +// possible in the Go code is worth a small (but measurable) performance boost +// by avoiding some MOVQs. Vars are needed for the asm and also are useful for +// convenience in the Go code in a few places where we need to intentionally +// avoid constant arithmetic (e.g., v1 := prime1 + prime2 fails because the +// result overflows a uint64). +var ( + prime1v = prime1 + prime2v = prime2 + prime3v = prime3 + prime4v = prime4 + prime5v = prime5 +) + +// Digest implements hash.Hash64. +type Digest struct { + v1 uint64 + v2 uint64 + v3 uint64 + v4 uint64 + total uint64 + mem [32]byte + n int // how much of mem is used +} + +// New creates a new Digest that computes the 64-bit xxHash algorithm. +func New() *Digest { + var d Digest + d.Reset() + return &d +} + +// Reset clears the Digest's state so that it can be reused. +func (d *Digest) Reset() { + d.v1 = prime1v + prime2 + d.v2 = prime2 + d.v3 = 0 + d.v4 = -prime1v + d.total = 0 + d.n = 0 +} + +// Size always returns 8 bytes. +func (d *Digest) Size() int { return 8 } + +// BlockSize always returns 32 bytes. +func (d *Digest) BlockSize() int { return 32 } + +// Write adds more data to d. It always returns len(b), nil. +func (d *Digest) Write(b []byte) (n int, err error) { + n = len(b) + d.total += uint64(n) + + if d.n+n < 32 { + // This new data doesn't even fill the current block. + copy(d.mem[d.n:], b) + d.n += n + return + } + + if d.n > 0 { + // Finish off the partial block. + copy(d.mem[d.n:], b) + d.v1 = round(d.v1, u64(d.mem[0:8])) + d.v2 = round(d.v2, u64(d.mem[8:16])) + d.v3 = round(d.v3, u64(d.mem[16:24])) + d.v4 = round(d.v4, u64(d.mem[24:32])) + b = b[32-d.n:] + d.n = 0 + } + + if len(b) >= 32 { + // One or more full blocks left. + nw := writeBlocks(d, b) + b = b[nw:] + } + + // Store any remaining partial block. + copy(d.mem[:], b) + d.n = len(b) + + return +} + +// Sum appends the current hash to b and returns the resulting slice. +func (d *Digest) Sum(b []byte) []byte { + s := d.Sum64() + return append( + b, + byte(s>>56), + byte(s>>48), + byte(s>>40), + byte(s>>32), + byte(s>>24), + byte(s>>16), + byte(s>>8), + byte(s), + ) +} + +// Sum64 returns the current hash. +func (d *Digest) Sum64() uint64 { + var h uint64 + + if d.total >= 32 { + v1, v2, v3, v4 := d.v1, d.v2, d.v3, d.v4 + h = rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4) + h = mergeRound(h, v1) + h = mergeRound(h, v2) + h = mergeRound(h, v3) + h = mergeRound(h, v4) + } else { + h = d.v3 + prime5 + } + + h += d.total + + i, end := 0, d.n + for ; i+8 <= end; i += 8 { + k1 := round(0, u64(d.mem[i:i+8])) + h ^= k1 + h = rol27(h)*prime1 + prime4 + } + if i+4 <= end { + h ^= uint64(u32(d.mem[i:i+4])) * prime1 + h = rol23(h)*prime2 + prime3 + i += 4 + } + for i < end { + h ^= uint64(d.mem[i]) * prime5 + h = rol11(h) * prime1 + i++ + } + + h ^= h >> 33 + h *= prime2 + h ^= h >> 29 + h *= prime3 + h ^= h >> 32 + + return h +} + +const ( + magic = "xxh\x06" + marshaledSize = len(magic) + 8*5 + 32 +) + +// MarshalBinary implements the encoding.BinaryMarshaler interface. +func (d *Digest) MarshalBinary() ([]byte, error) { + b := make([]byte, 0, marshaledSize) + b = append(b, magic...) + b = appendUint64(b, d.v1) + b = appendUint64(b, d.v2) + b = appendUint64(b, d.v3) + b = appendUint64(b, d.v4) + b = appendUint64(b, d.total) + b = append(b, d.mem[:d.n]...) + b = b[:len(b)+len(d.mem)-d.n] + return b, nil +} + +// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface. +func (d *Digest) UnmarshalBinary(b []byte) error { + if len(b) < len(magic) || string(b[:len(magic)]) != magic { + return errors.New("xxhash: invalid hash state identifier") + } + if len(b) != marshaledSize { + return errors.New("xxhash: invalid hash state size") + } + b = b[len(magic):] + b, d.v1 = consumeUint64(b) + b, d.v2 = consumeUint64(b) + b, d.v3 = consumeUint64(b) + b, d.v4 = consumeUint64(b) + b, d.total = consumeUint64(b) + copy(d.mem[:], b) + b = b[len(d.mem):] + d.n = int(d.total % uint64(len(d.mem))) + return nil +} + +func appendUint64(b []byte, x uint64) []byte { + var a [8]byte + binary.LittleEndian.PutUint64(a[:], x) + return append(b, a[:]...) +} + +func consumeUint64(b []byte) ([]byte, uint64) { + x := u64(b) + return b[8:], x +} + +func u64(b []byte) uint64 { return binary.LittleEndian.Uint64(b) } +func u32(b []byte) uint32 { return binary.LittleEndian.Uint32(b) } + +func round(acc, input uint64) uint64 { + acc += input * prime2 + acc = rol31(acc) + acc *= prime1 + return acc +} + +func mergeRound(acc, val uint64) uint64 { + val = round(0, val) + acc ^= val + acc = acc*prime1 + prime4 + return acc +} + +func rol1(x uint64) uint64 { return bits.RotateLeft64(x, 1) } +func rol7(x uint64) uint64 { return bits.RotateLeft64(x, 7) } +func rol11(x uint64) uint64 { return bits.RotateLeft64(x, 11) } +func rol12(x uint64) uint64 { return bits.RotateLeft64(x, 12) } +func rol18(x uint64) uint64 { return bits.RotateLeft64(x, 18) } +func rol23(x uint64) uint64 { return bits.RotateLeft64(x, 23) } +func rol27(x uint64) uint64 { return bits.RotateLeft64(x, 27) } +func rol31(x uint64) uint64 { return bits.RotateLeft64(x, 31) } diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.go b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.go new file mode 100644 index 000000000..35318d7c4 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.go @@ -0,0 +1,13 @@ +// +build !appengine +// +build gc +// +build !purego + +package xxhash + +// Sum64 computes the 64-bit xxHash digest of b. +// +//go:noescape +func Sum64(b []byte) uint64 + +//go:noescape +func writeBlocks(*Digest, []byte) int diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s new file mode 100644 index 000000000..d580e32ae --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s @@ -0,0 +1,215 @@ +// +build !appengine +// +build gc +// +build !purego + +#include "textflag.h" + +// Register allocation: +// AX h +// CX pointer to advance through b +// DX n +// BX loop end +// R8 v1, k1 +// R9 v2 +// R10 v3 +// R11 v4 +// R12 tmp +// R13 prime1v +// R14 prime2v +// R15 prime4v + +// round reads from and advances the buffer pointer in CX. +// It assumes that R13 has prime1v and R14 has prime2v. +#define round(r) \ + MOVQ (CX), R12 \ + ADDQ $8, CX \ + IMULQ R14, R12 \ + ADDQ R12, r \ + ROLQ $31, r \ + IMULQ R13, r + +// mergeRound applies a merge round on the two registers acc and val. +// It assumes that R13 has prime1v, R14 has prime2v, and R15 has prime4v. +#define mergeRound(acc, val) \ + IMULQ R14, val \ + ROLQ $31, val \ + IMULQ R13, val \ + XORQ val, acc \ + IMULQ R13, acc \ + ADDQ R15, acc + +// func Sum64(b []byte) uint64 +TEXT ·Sum64(SB), NOSPLIT, $0-32 + // Load fixed primes. + MOVQ ·prime1v(SB), R13 + MOVQ ·prime2v(SB), R14 + MOVQ ·prime4v(SB), R15 + + // Load slice. + MOVQ b_base+0(FP), CX + MOVQ b_len+8(FP), DX + LEAQ (CX)(DX*1), BX + + // The first loop limit will be len(b)-32. + SUBQ $32, BX + + // Check whether we have at least one block. + CMPQ DX, $32 + JLT noBlocks + + // Set up initial state (v1, v2, v3, v4). + MOVQ R13, R8 + ADDQ R14, R8 + MOVQ R14, R9 + XORQ R10, R10 + XORQ R11, R11 + SUBQ R13, R11 + + // Loop until CX > BX. +blockLoop: + round(R8) + round(R9) + round(R10) + round(R11) + + CMPQ CX, BX + JLE blockLoop + + MOVQ R8, AX + ROLQ $1, AX + MOVQ R9, R12 + ROLQ $7, R12 + ADDQ R12, AX + MOVQ R10, R12 + ROLQ $12, R12 + ADDQ R12, AX + MOVQ R11, R12 + ROLQ $18, R12 + ADDQ R12, AX + + mergeRound(AX, R8) + mergeRound(AX, R9) + mergeRound(AX, R10) + mergeRound(AX, R11) + + JMP afterBlocks + +noBlocks: + MOVQ ·prime5v(SB), AX + +afterBlocks: + ADDQ DX, AX + + // Right now BX has len(b)-32, and we want to loop until CX > len(b)-8. + ADDQ $24, BX + + CMPQ CX, BX + JG fourByte + +wordLoop: + // Calculate k1. + MOVQ (CX), R8 + ADDQ $8, CX + IMULQ R14, R8 + ROLQ $31, R8 + IMULQ R13, R8 + + XORQ R8, AX + ROLQ $27, AX + IMULQ R13, AX + ADDQ R15, AX + + CMPQ CX, BX + JLE wordLoop + +fourByte: + ADDQ $4, BX + CMPQ CX, BX + JG singles + + MOVL (CX), R8 + ADDQ $4, CX + IMULQ R13, R8 + XORQ R8, AX + + ROLQ $23, AX + IMULQ R14, AX + ADDQ ·prime3v(SB), AX + +singles: + ADDQ $4, BX + CMPQ CX, BX + JGE finalize + +singlesLoop: + MOVBQZX (CX), R12 + ADDQ $1, CX + IMULQ ·prime5v(SB), R12 + XORQ R12, AX + + ROLQ $11, AX + IMULQ R13, AX + + CMPQ CX, BX + JL singlesLoop + +finalize: + MOVQ AX, R12 + SHRQ $33, R12 + XORQ R12, AX + IMULQ R14, AX + MOVQ AX, R12 + SHRQ $29, R12 + XORQ R12, AX + IMULQ ·prime3v(SB), AX + MOVQ AX, R12 + SHRQ $32, R12 + XORQ R12, AX + + MOVQ AX, ret+24(FP) + RET + +// writeBlocks uses the same registers as above except that it uses AX to store +// the d pointer. + +// func writeBlocks(d *Digest, b []byte) int +TEXT ·writeBlocks(SB), NOSPLIT, $0-40 + // Load fixed primes needed for round. + MOVQ ·prime1v(SB), R13 + MOVQ ·prime2v(SB), R14 + + // Load slice. + MOVQ b_base+8(FP), CX + MOVQ b_len+16(FP), DX + LEAQ (CX)(DX*1), BX + SUBQ $32, BX + + // Load vN from d. + MOVQ d+0(FP), AX + MOVQ 0(AX), R8 // v1 + MOVQ 8(AX), R9 // v2 + MOVQ 16(AX), R10 // v3 + MOVQ 24(AX), R11 // v4 + + // We don't need to check the loop condition here; this function is + // always called with at least one block of data to process. +blockLoop: + round(R8) + round(R9) + round(R10) + round(R11) + + CMPQ CX, BX + JLE blockLoop + + // Copy vN back to d. + MOVQ R8, 0(AX) + MOVQ R9, 8(AX) + MOVQ R10, 16(AX) + MOVQ R11, 24(AX) + + // The number of bytes written is CX minus the old base pointer. + SUBQ b_base+8(FP), CX + MOVQ CX, ret+32(FP) + + RET diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go new file mode 100644 index 000000000..4a5a82160 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go @@ -0,0 +1,76 @@ +// +build !amd64 appengine !gc purego + +package xxhash + +// Sum64 computes the 64-bit xxHash digest of b. +func Sum64(b []byte) uint64 { + // A simpler version would be + // d := New() + // d.Write(b) + // return d.Sum64() + // but this is faster, particularly for small inputs. + + n := len(b) + var h uint64 + + if n >= 32 { + v1 := prime1v + prime2 + v2 := prime2 + v3 := uint64(0) + v4 := -prime1v + for len(b) >= 32 { + v1 = round(v1, u64(b[0:8:len(b)])) + v2 = round(v2, u64(b[8:16:len(b)])) + v3 = round(v3, u64(b[16:24:len(b)])) + v4 = round(v4, u64(b[24:32:len(b)])) + b = b[32:len(b):len(b)] + } + h = rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4) + h = mergeRound(h, v1) + h = mergeRound(h, v2) + h = mergeRound(h, v3) + h = mergeRound(h, v4) + } else { + h = prime5 + } + + h += uint64(n) + + i, end := 0, len(b) + for ; i+8 <= end; i += 8 { + k1 := round(0, u64(b[i:i+8:len(b)])) + h ^= k1 + h = rol27(h)*prime1 + prime4 + } + if i+4 <= end { + h ^= uint64(u32(b[i:i+4:len(b)])) * prime1 + h = rol23(h)*prime2 + prime3 + i += 4 + } + for ; i < end; i++ { + h ^= uint64(b[i]) * prime5 + h = rol11(h) * prime1 + } + + h ^= h >> 33 + h *= prime2 + h ^= h >> 29 + h *= prime3 + h ^= h >> 32 + + return h +} + +func writeBlocks(d *Digest, b []byte) int { + v1, v2, v3, v4 := d.v1, d.v2, d.v3, d.v4 + n := len(b) + for len(b) >= 32 { + v1 = round(v1, u64(b[0:8:len(b)])) + v2 = round(v2, u64(b[8:16:len(b)])) + v3 = round(v3, u64(b[16:24:len(b)])) + v4 = round(v4, u64(b[24:32:len(b)])) + b = b[32:len(b):len(b)] + } + d.v1, d.v2, d.v3, d.v4 = v1, v2, v3, v4 + return n - len(b) +} diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_safe.go b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_safe.go new file mode 100644 index 000000000..6f3b0cb10 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_safe.go @@ -0,0 +1,11 @@ +package xxhash + +// Sum64String computes the 64-bit xxHash digest of s. +func Sum64String(s string) uint64 { + return Sum64([]byte(s)) +} + +// WriteString adds more data to d. It always returns len(s), nil. +func (d *Digest) WriteString(s string) (n int, err error) { + return d.Write([]byte(s)) +} diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec.go b/vendor/github.com/klauspost/compress/zstd/seqdec.go new file mode 100644 index 000000000..15a45f7b5 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/seqdec.go @@ -0,0 +1,402 @@ +// Copyright 2019+ Klaus Post. All rights reserved. +// License information can be found in the LICENSE file. +// Based on work by Yann Collet, released under BSD License. + +package zstd + +import ( + "errors" + "fmt" + "io" +) + +type seq struct { + litLen uint32 + matchLen uint32 + offset uint32 + + // Codes are stored here for the encoder + // so they only have to be looked up once. + llCode, mlCode, ofCode uint8 +} + +func (s seq) String() string { + if s.offset <= 3 { + if s.offset == 0 { + return fmt.Sprint("litLen:", s.litLen, ", matchLen:", s.matchLen+zstdMinMatch, ", offset: INVALID (0)") + } + return fmt.Sprint("litLen:", s.litLen, ", matchLen:", s.matchLen+zstdMinMatch, ", offset:", s.offset, " (repeat)") + } + return fmt.Sprint("litLen:", s.litLen, ", matchLen:", s.matchLen+zstdMinMatch, ", offset:", s.offset-3, " (new)") +} + +type seqCompMode uint8 + +const ( + compModePredefined seqCompMode = iota + compModeRLE + compModeFSE + compModeRepeat +) + +type sequenceDec struct { + // decoder keeps track of the current state and updates it from the bitstream. + fse *fseDecoder + state fseState + repeat bool +} + +// init the state of the decoder with input from stream. +func (s *sequenceDec) init(br *bitReader) error { + if s.fse == nil { + return errors.New("sequence decoder not defined") + } + s.state.init(br, s.fse.actualTableLog, s.fse.dt[:1<= 0; i-- { + if br.overread() { + printf("reading sequence %d, exceeded available data\n", seqs-i) + return io.ErrUnexpectedEOF + } + var litLen, matchOff, matchLen int + if br.off > 4+((maxOffsetBits+16+16)>>3) { + litLen, matchOff, matchLen = s.nextFast(br, llState, mlState, ofState) + br.fillFast() + } else { + litLen, matchOff, matchLen = s.next(br, llState, mlState, ofState) + br.fill() + } + + if debugSequences { + println("Seq", seqs-i-1, "Litlen:", litLen, "matchOff:", matchOff, "(abs) matchLen:", matchLen) + } + + if litLen > len(s.literals) { + return fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available", litLen, len(s.literals)) + } + size := litLen + matchLen + len(s.out) + if size-startSize > maxBlockSize { + return fmt.Errorf("output (%d) bigger than max block size", size) + } + if size > cap(s.out) { + // Not enough size, will be extremely rarely triggered, + // but could be if destination slice is too small for sync operations. + // We add maxBlockSize to the capacity. + s.out = append(s.out, make([]byte, maxBlockSize)...) + s.out = s.out[:len(s.out)-maxBlockSize] + } + if matchLen > maxMatchLen { + return fmt.Errorf("match len (%d) bigger than max allowed length", matchLen) + } + if matchOff > len(s.out)+len(hist)+litLen { + return fmt.Errorf("match offset (%d) bigger than current history (%d)", matchOff, len(s.out)+len(hist)+litLen) + } + if matchOff == 0 && matchLen > 0 { + return fmt.Errorf("zero matchoff and matchlen > 0") + } + + s.out = append(s.out, s.literals[:litLen]...) + s.literals = s.literals[litLen:] + out := s.out + + // Copy from history. + // TODO: Blocks without history could be made to ignore this completely. + if v := matchOff - len(s.out); v > 0 { + // v is the start position in history from end. + start := len(s.hist) - v + if matchLen > v { + // Some goes into current block. + // Copy remainder of history + out = append(out, s.hist[start:]...) + matchOff -= v + matchLen -= v + } else { + out = append(out, s.hist[start:start+matchLen]...) + matchLen = 0 + } + } + // We must be in current buffer now + if matchLen > 0 { + start := len(s.out) - matchOff + if matchLen <= len(s.out)-start { + // No overlap + out = append(out, s.out[start:start+matchLen]...) + } else { + // Overlapping copy + // Extend destination slice and copy one byte at the time. + out = out[:len(out)+matchLen] + src := out[start : start+matchLen] + // Destination is the space we just added. + dst := out[len(out)-matchLen:] + dst = dst[:len(src)] + for i := range src { + dst[i] = src[i] + } + } + } + s.out = out + if i == 0 { + // This is the last sequence, so we shouldn't update state. + break + } + + // Manually inlined, ~ 5-20% faster + // Update all 3 states at once. Approx 20% faster. + nBits := llState.nbBits() + mlState.nbBits() + ofState.nbBits() + if nBits == 0 { + llState = llTable[llState.newState()&maxTableMask] + mlState = mlTable[mlState.newState()&maxTableMask] + ofState = ofTable[ofState.newState()&maxTableMask] + } else { + bits := br.getBitsFast(nBits) + lowBits := uint16(bits >> ((ofState.nbBits() + mlState.nbBits()) & 31)) + llState = llTable[(llState.newState()+lowBits)&maxTableMask] + + lowBits = uint16(bits >> (ofState.nbBits() & 31)) + lowBits &= bitMask[mlState.nbBits()&15] + mlState = mlTable[(mlState.newState()+lowBits)&maxTableMask] + + lowBits = uint16(bits) & bitMask[ofState.nbBits()&15] + ofState = ofTable[(ofState.newState()+lowBits)&maxTableMask] + } + } + + // Add final literals + s.out = append(s.out, s.literals...) + return nil +} + +// update states, at least 27 bits must be available. +func (s *sequenceDecs) update(br *bitReader) { + // Max 8 bits + s.litLengths.state.next(br) + // Max 9 bits + s.matchLengths.state.next(br) + // Max 8 bits + s.offsets.state.next(br) +} + +var bitMask [16]uint16 + +func init() { + for i := range bitMask[:] { + bitMask[i] = uint16((1 << uint(i)) - 1) + } +} + +// update states, at least 27 bits must be available. +func (s *sequenceDecs) updateAlt(br *bitReader) { + // Update all 3 states at once. Approx 20% faster. + a, b, c := s.litLengths.state.state, s.matchLengths.state.state, s.offsets.state.state + + nBits := a.nbBits() + b.nbBits() + c.nbBits() + if nBits == 0 { + s.litLengths.state.state = s.litLengths.state.dt[a.newState()] + s.matchLengths.state.state = s.matchLengths.state.dt[b.newState()] + s.offsets.state.state = s.offsets.state.dt[c.newState()] + return + } + bits := br.getBitsFast(nBits) + lowBits := uint16(bits >> ((c.nbBits() + b.nbBits()) & 31)) + s.litLengths.state.state = s.litLengths.state.dt[a.newState()+lowBits] + + lowBits = uint16(bits >> (c.nbBits() & 31)) + lowBits &= bitMask[b.nbBits()&15] + s.matchLengths.state.state = s.matchLengths.state.dt[b.newState()+lowBits] + + lowBits = uint16(bits) & bitMask[c.nbBits()&15] + s.offsets.state.state = s.offsets.state.dt[c.newState()+lowBits] +} + +// nextFast will return new states when there are at least 4 unused bytes left on the stream when done. +func (s *sequenceDecs) nextFast(br *bitReader, llState, mlState, ofState decSymbol) (ll, mo, ml int) { + // Final will not read from stream. + ll, llB := llState.final() + ml, mlB := mlState.final() + mo, moB := ofState.final() + + // extra bits are stored in reverse order. + br.fillFast() + mo += br.getBits(moB) + if s.maxBits > 32 { + br.fillFast() + } + ml += br.getBits(mlB) + ll += br.getBits(llB) + + if moB > 1 { + s.prevOffset[2] = s.prevOffset[1] + s.prevOffset[1] = s.prevOffset[0] + s.prevOffset[0] = mo + return + } + // mo = s.adjustOffset(mo, ll, moB) + // Inlined for rather big speedup + if ll == 0 { + // There is an exception though, when current sequence's literals_length = 0. + // In this case, repeated offsets are shifted by one, so an offset_value of 1 means Repeated_Offset2, + // an offset_value of 2 means Repeated_Offset3, and an offset_value of 3 means Repeated_Offset1 - 1_byte. + mo++ + } + + if mo == 0 { + mo = s.prevOffset[0] + return + } + var temp int + if mo == 3 { + temp = s.prevOffset[0] - 1 + } else { + temp = s.prevOffset[mo] + } + + if temp == 0 { + // 0 is not valid; input is corrupted; force offset to 1 + println("temp was 0") + temp = 1 + } + + if mo != 1 { + s.prevOffset[2] = s.prevOffset[1] + } + s.prevOffset[1] = s.prevOffset[0] + s.prevOffset[0] = temp + mo = temp + return +} + +func (s *sequenceDecs) next(br *bitReader, llState, mlState, ofState decSymbol) (ll, mo, ml int) { + // Final will not read from stream. + ll, llB := llState.final() + ml, mlB := mlState.final() + mo, moB := ofState.final() + + // extra bits are stored in reverse order. + br.fill() + if s.maxBits <= 32 { + mo += br.getBits(moB) + ml += br.getBits(mlB) + ll += br.getBits(llB) + } else { + mo += br.getBits(moB) + br.fill() + // matchlength+literal length, max 32 bits + ml += br.getBits(mlB) + ll += br.getBits(llB) + + } + mo = s.adjustOffset(mo, ll, moB) + return +} + +func (s *sequenceDecs) adjustOffset(offset, litLen int, offsetB uint8) int { + if offsetB > 1 { + s.prevOffset[2] = s.prevOffset[1] + s.prevOffset[1] = s.prevOffset[0] + s.prevOffset[0] = offset + return offset + } + + if litLen == 0 { + // There is an exception though, when current sequence's literals_length = 0. + // In this case, repeated offsets are shifted by one, so an offset_value of 1 means Repeated_Offset2, + // an offset_value of 2 means Repeated_Offset3, and an offset_value of 3 means Repeated_Offset1 - 1_byte. + offset++ + } + + if offset == 0 { + return s.prevOffset[0] + } + var temp int + if offset == 3 { + temp = s.prevOffset[0] - 1 + } else { + temp = s.prevOffset[offset] + } + + if temp == 0 { + // 0 is not valid; input is corrupted; force offset to 1 + println("temp was 0") + temp = 1 + } + + if offset != 1 { + s.prevOffset[2] = s.prevOffset[1] + } + s.prevOffset[1] = s.prevOffset[0] + s.prevOffset[0] = temp + return temp +} + +// mergeHistory will merge history. +func (s *sequenceDecs) mergeHistory(hist *sequenceDecs) (*sequenceDecs, error) { + for i := uint(0); i < 3; i++ { + var sNew, sHist *sequenceDec + switch i { + default: + // same as "case 0": + sNew = &s.litLengths + sHist = &hist.litLengths + case 1: + sNew = &s.offsets + sHist = &hist.offsets + case 2: + sNew = &s.matchLengths + sHist = &hist.matchLengths + } + if sNew.repeat { + if sHist.fse == nil { + return nil, fmt.Errorf("sequence stream %d, repeat requested, but no history", i) + } + continue + } + if sNew.fse == nil { + return nil, fmt.Errorf("sequence stream %d, no fse found", i) + } + if sHist.fse != nil && !sHist.fse.preDefined { + fseDecoderPool.Put(sHist.fse) + } + sHist.fse = sNew.fse + } + return hist, nil +} diff --git a/vendor/github.com/klauspost/compress/zstd/seqenc.go b/vendor/github.com/klauspost/compress/zstd/seqenc.go new file mode 100644 index 000000000..36bcc3cc0 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/seqenc.go @@ -0,0 +1,115 @@ +// Copyright 2019+ Klaus Post. All rights reserved. +// License information can be found in the LICENSE file. +// Based on work by Yann Collet, released under BSD License. + +package zstd + +import "math/bits" + +type seqCoders struct { + llEnc, ofEnc, mlEnc *fseEncoder + llPrev, ofPrev, mlPrev *fseEncoder +} + +// swap coders with another (block). +func (s *seqCoders) swap(other *seqCoders) { + *s, *other = *other, *s +} + +// setPrev will update the previous encoders to the actually used ones +// and make sure a fresh one is in the main slot. +func (s *seqCoders) setPrev(ll, ml, of *fseEncoder) { + compareSwap := func(used *fseEncoder, current, prev **fseEncoder) { + // We used the new one, more current to history and reuse the previous history + if *current == used { + *prev, *current = *current, *prev + c := *current + p := *prev + c.reUsed = false + p.reUsed = true + return + } + if used == *prev { + return + } + // Ensure we cannot reuse by accident + prevEnc := *prev + prevEnc.symbolLen = 0 + return + } + compareSwap(ll, &s.llEnc, &s.llPrev) + compareSwap(ml, &s.mlEnc, &s.mlPrev) + compareSwap(of, &s.ofEnc, &s.ofPrev) +} + +func highBit(val uint32) (n uint32) { + return uint32(bits.Len32(val) - 1) +} + +var llCodeTable = [64]byte{0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 16, 17, 17, 18, 18, 19, 19, + 20, 20, 20, 20, 21, 21, 21, 21, + 22, 22, 22, 22, 22, 22, 22, 22, + 23, 23, 23, 23, 23, 23, 23, 23, + 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24} + +// Up to 6 bits +const maxLLCode = 35 + +// llBitsTable translates from ll code to number of bits. +var llBitsTable = [maxLLCode + 1]byte{ + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 2, 2, 3, 3, + 4, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 16} + +// llCode returns the code that represents the literal length requested. +func llCode(litLength uint32) uint8 { + const llDeltaCode = 19 + if litLength <= 63 { + // Compiler insists on bounds check (Go 1.12) + return llCodeTable[litLength&63] + } + return uint8(highBit(litLength)) + llDeltaCode +} + +var mlCodeTable = [128]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, + 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42} + +// Up to 6 bits +const maxMLCode = 52 + +// mlBitsTable translates from ml code to number of bits. +var mlBitsTable = [maxMLCode + 1]byte{ + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 2, 2, 3, 3, + 4, 4, 5, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 16} + +// note : mlBase = matchLength - MINMATCH; +// because it's the format it's stored in seqStore->sequences +func mlCode(mlBase uint32) uint8 { + const mlDeltaCode = 36 + if mlBase <= 127 { + // Compiler insists on bounds check (Go 1.12) + return mlCodeTable[mlBase&127] + } + return uint8(highBit(mlBase)) + mlDeltaCode +} + +func ofCode(offset uint32) uint8 { + // A valid offset will always be > 0. + return uint8(bits.Len32(offset) - 1) +} diff --git a/vendor/github.com/klauspost/compress/zstd/snappy.go b/vendor/github.com/klauspost/compress/zstd/snappy.go new file mode 100644 index 000000000..356956ba2 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/snappy.go @@ -0,0 +1,436 @@ +// Copyright 2019+ Klaus Post. All rights reserved. +// License information can be found in the LICENSE file. +// Based on work by Yann Collet, released under BSD License. + +package zstd + +import ( + "encoding/binary" + "errors" + "hash/crc32" + "io" + + "github.com/klauspost/compress/huff0" + "github.com/klauspost/compress/snappy" +) + +const ( + snappyTagLiteral = 0x00 + snappyTagCopy1 = 0x01 + snappyTagCopy2 = 0x02 + snappyTagCopy4 = 0x03 +) + +const ( + snappyChecksumSize = 4 + snappyMagicBody = "sNaPpY" + + // snappyMaxBlockSize is the maximum size of the input to encodeBlock. It is not + // part of the wire format per se, but some parts of the encoder assume + // that an offset fits into a uint16. + // + // Also, for the framing format (Writer type instead of Encode function), + // https://github.com/google/snappy/blob/master/framing_format.txt says + // that "the uncompressed data in a chunk must be no longer than 65536 + // bytes". + snappyMaxBlockSize = 65536 + + // snappyMaxEncodedLenOfMaxBlockSize equals MaxEncodedLen(snappyMaxBlockSize), but is + // hard coded to be a const instead of a variable, so that obufLen can also + // be a const. Their equivalence is confirmed by + // TestMaxEncodedLenOfMaxBlockSize. + snappyMaxEncodedLenOfMaxBlockSize = 76490 +) + +const ( + chunkTypeCompressedData = 0x00 + chunkTypeUncompressedData = 0x01 + chunkTypePadding = 0xfe + chunkTypeStreamIdentifier = 0xff +) + +var ( + // ErrSnappyCorrupt reports that the input is invalid. + ErrSnappyCorrupt = errors.New("snappy: corrupt input") + // ErrSnappyTooLarge reports that the uncompressed length is too large. + ErrSnappyTooLarge = errors.New("snappy: decoded block is too large") + // ErrSnappyUnsupported reports that the input isn't supported. + ErrSnappyUnsupported = errors.New("snappy: unsupported input") + + errUnsupportedLiteralLength = errors.New("snappy: unsupported literal length") +) + +// SnappyConverter can read SnappyConverter-compressed streams and convert them to zstd. +// Conversion is done by converting the stream directly from Snappy without intermediate +// full decoding. +// Therefore the compression ratio is much less than what can be done by a full decompression +// and compression, and a faulty Snappy stream may lead to a faulty Zstandard stream without +// any errors being generated. +// No CRC value is being generated and not all CRC values of the Snappy stream are checked. +// However, it provides really fast recompression of Snappy streams. +// The converter can be reused to avoid allocations, even after errors. +type SnappyConverter struct { + r io.Reader + err error + buf []byte + block *blockEnc +} + +// Convert the Snappy stream supplied in 'in' and write the zStandard stream to 'w'. +// If any error is detected on the Snappy stream it is returned. +// The number of bytes written is returned. +func (r *SnappyConverter) Convert(in io.Reader, w io.Writer) (int64, error) { + initPredefined() + r.err = nil + r.r = in + if r.block == nil { + r.block = &blockEnc{} + r.block.init() + } + r.block.initNewEncode() + if len(r.buf) != snappyMaxEncodedLenOfMaxBlockSize+snappyChecksumSize { + r.buf = make([]byte, snappyMaxEncodedLenOfMaxBlockSize+snappyChecksumSize) + } + r.block.litEnc.Reuse = huff0.ReusePolicyNone + var written int64 + var readHeader bool + { + var header []byte + var n int + header, r.err = frameHeader{WindowSize: snappyMaxBlockSize}.appendTo(r.buf[:0]) + + n, r.err = w.Write(header) + if r.err != nil { + return written, r.err + } + written += int64(n) + } + + for { + if !r.readFull(r.buf[:4], true) { + // Add empty last block + r.block.reset(nil) + r.block.last = true + err := r.block.encodeLits(false) + if err != nil { + return written, err + } + n, err := w.Write(r.block.output) + if err != nil { + return written, err + } + written += int64(n) + + return written, r.err + } + chunkType := r.buf[0] + if !readHeader { + if chunkType != chunkTypeStreamIdentifier { + println("chunkType != chunkTypeStreamIdentifier", chunkType) + r.err = ErrSnappyCorrupt + return written, r.err + } + readHeader = true + } + chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16 + if chunkLen > len(r.buf) { + println("chunkLen > len(r.buf)", chunkType) + r.err = ErrSnappyUnsupported + return written, r.err + } + + // The chunk types are specified at + // https://github.com/google/snappy/blob/master/framing_format.txt + switch chunkType { + case chunkTypeCompressedData: + // Section 4.2. Compressed data (chunk type 0x00). + if chunkLen < snappyChecksumSize { + println("chunkLen < snappyChecksumSize", chunkLen, snappyChecksumSize) + r.err = ErrSnappyCorrupt + return written, r.err + } + buf := r.buf[:chunkLen] + if !r.readFull(buf, false) { + return written, r.err + } + //checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 + buf = buf[snappyChecksumSize:] + + n, hdr, err := snappyDecodedLen(buf) + if err != nil { + r.err = err + return written, r.err + } + buf = buf[hdr:] + if n > snappyMaxBlockSize { + println("n > snappyMaxBlockSize", n, snappyMaxBlockSize) + r.err = ErrSnappyCorrupt + return written, r.err + } + r.block.reset(nil) + r.block.pushOffsets() + if err := decodeSnappy(r.block, buf); err != nil { + r.err = err + return written, r.err + } + if r.block.size+r.block.extraLits != n { + printf("invalid size, want %d, got %d\n", n, r.block.size+r.block.extraLits) + r.err = ErrSnappyCorrupt + return written, r.err + } + err = r.block.encode(false) + switch err { + case errIncompressible: + r.block.popOffsets() + r.block.reset(nil) + r.block.literals, err = snappy.Decode(r.block.literals[:n], r.buf[snappyChecksumSize:chunkLen]) + if err != nil { + println("snappy.Decode:", err) + return written, err + } + err = r.block.encodeLits(false) + if err != nil { + return written, err + } + case nil: + default: + return written, err + } + + n, r.err = w.Write(r.block.output) + if r.err != nil { + return written, err + } + written += int64(n) + continue + case chunkTypeUncompressedData: + if debug { + println("Uncompressed, chunklen", chunkLen) + } + // Section 4.3. Uncompressed data (chunk type 0x01). + if chunkLen < snappyChecksumSize { + println("chunkLen < snappyChecksumSize", chunkLen, snappyChecksumSize) + r.err = ErrSnappyCorrupt + return written, r.err + } + r.block.reset(nil) + buf := r.buf[:snappyChecksumSize] + if !r.readFull(buf, false) { + return written, r.err + } + checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 + // Read directly into r.decoded instead of via r.buf. + n := chunkLen - snappyChecksumSize + if n > snappyMaxBlockSize { + println("n > snappyMaxBlockSize", n, snappyMaxBlockSize) + r.err = ErrSnappyCorrupt + return written, r.err + } + r.block.literals = r.block.literals[:n] + if !r.readFull(r.block.literals, false) { + return written, r.err + } + if snappyCRC(r.block.literals) != checksum { + println("literals crc mismatch") + r.err = ErrSnappyCorrupt + return written, r.err + } + err := r.block.encodeLits(false) + if err != nil { + return written, err + } + n, r.err = w.Write(r.block.output) + if r.err != nil { + return written, err + } + written += int64(n) + continue + + case chunkTypeStreamIdentifier: + if debug { + println("stream id", chunkLen, len(snappyMagicBody)) + } + // Section 4.1. Stream identifier (chunk type 0xff). + if chunkLen != len(snappyMagicBody) { + println("chunkLen != len(snappyMagicBody)", chunkLen, len(snappyMagicBody)) + r.err = ErrSnappyCorrupt + return written, r.err + } + if !r.readFull(r.buf[:len(snappyMagicBody)], false) { + return written, r.err + } + for i := 0; i < len(snappyMagicBody); i++ { + if r.buf[i] != snappyMagicBody[i] { + println("r.buf[i] != snappyMagicBody[i]", r.buf[i], snappyMagicBody[i], i) + r.err = ErrSnappyCorrupt + return written, r.err + } + } + continue + } + + if chunkType <= 0x7f { + // Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f). + println("chunkType <= 0x7f") + r.err = ErrSnappyUnsupported + return written, r.err + } + // Section 4.4 Padding (chunk type 0xfe). + // Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd). + if !r.readFull(r.buf[:chunkLen], false) { + return written, r.err + } + } +} + +// decodeSnappy writes the decoding of src to dst. It assumes that the varint-encoded +// length of the decompressed bytes has already been read. +func decodeSnappy(blk *blockEnc, src []byte) error { + //decodeRef(make([]byte, snappyMaxBlockSize), src) + var s, length int + lits := blk.extraLits + var offset uint32 + for s < len(src) { + switch src[s] & 0x03 { + case snappyTagLiteral: + x := uint32(src[s] >> 2) + switch { + case x < 60: + s++ + case x == 60: + s += 2 + if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. + println("uint(s) > uint(len(src)", s, src) + return ErrSnappyCorrupt + } + x = uint32(src[s-1]) + case x == 61: + s += 3 + if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. + println("uint(s) > uint(len(src)", s, src) + return ErrSnappyCorrupt + } + x = uint32(src[s-2]) | uint32(src[s-1])<<8 + case x == 62: + s += 4 + if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. + println("uint(s) > uint(len(src)", s, src) + return ErrSnappyCorrupt + } + x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16 + case x == 63: + s += 5 + if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. + println("uint(s) > uint(len(src)", s, src) + return ErrSnappyCorrupt + } + x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24 + } + if x > snappyMaxBlockSize { + println("x > snappyMaxBlockSize", x, snappyMaxBlockSize) + return ErrSnappyCorrupt + } + length = int(x) + 1 + if length <= 0 { + println("length <= 0 ", length) + + return errUnsupportedLiteralLength + } + //if length > snappyMaxBlockSize-d || uint32(length) > len(src)-s { + // return ErrSnappyCorrupt + //} + + blk.literals = append(blk.literals, src[s:s+length]...) + //println(length, "litLen") + lits += length + s += length + continue + + case snappyTagCopy1: + s += 2 + if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. + println("uint(s) > uint(len(src)", s, len(src)) + return ErrSnappyCorrupt + } + length = 4 + int(src[s-2])>>2&0x7 + offset = uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]) + + case snappyTagCopy2: + s += 3 + if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. + println("uint(s) > uint(len(src)", s, len(src)) + return ErrSnappyCorrupt + } + length = 1 + int(src[s-3])>>2 + offset = uint32(src[s-2]) | uint32(src[s-1])<<8 + + case snappyTagCopy4: + s += 5 + if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. + println("uint(s) > uint(len(src)", s, len(src)) + return ErrSnappyCorrupt + } + length = 1 + int(src[s-5])>>2 + offset = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24 + } + + if offset <= 0 || blk.size+lits < int(offset) /*|| length > len(blk)-d */ { + println("offset <= 0 || blk.size+lits < int(offset)", offset, blk.size+lits, int(offset), blk.size, lits) + + return ErrSnappyCorrupt + } + + // Check if offset is one of the recent offsets. + // Adjusts the output offset accordingly. + // Gives a tiny bit of compression, typically around 1%. + if false { + offset = blk.matchOffset(offset, uint32(lits)) + } else { + offset += 3 + } + + blk.sequences = append(blk.sequences, seq{ + litLen: uint32(lits), + offset: offset, + matchLen: uint32(length) - zstdMinMatch, + }) + blk.size += length + lits + lits = 0 + } + blk.extraLits = lits + return nil +} + +func (r *SnappyConverter) readFull(p []byte, allowEOF bool) (ok bool) { + if _, r.err = io.ReadFull(r.r, p); r.err != nil { + if r.err == io.ErrUnexpectedEOF || (r.err == io.EOF && !allowEOF) { + r.err = ErrSnappyCorrupt + } + return false + } + return true +} + +var crcTable = crc32.MakeTable(crc32.Castagnoli) + +// crc implements the checksum specified in section 3 of +// https://github.com/google/snappy/blob/master/framing_format.txt +func snappyCRC(b []byte) uint32 { + c := crc32.Update(0, crcTable, b) + return uint32(c>>15|c<<17) + 0xa282ead8 +} + +// snappyDecodedLen returns the length of the decoded block and the number of bytes +// that the length header occupied. +func snappyDecodedLen(src []byte) (blockLen, headerLen int, err error) { + v, n := binary.Uvarint(src) + if n <= 0 || v > 0xffffffff { + return 0, 0, ErrSnappyCorrupt + } + + const wordSize = 32 << (^uint(0) >> 32 & 1) + if wordSize == 32 && v > 0x7fffffff { + return 0, 0, ErrSnappyTooLarge + } + return int(v), n, nil +} diff --git a/vendor/github.com/klauspost/compress/zstd/zstd.go b/vendor/github.com/klauspost/compress/zstd/zstd.go new file mode 100644 index 000000000..57a8a2f5b --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/zstd.go @@ -0,0 +1,136 @@ +// Package zstd provides decompression of zstandard files. +// +// For advanced usage and examples, go to the README: https://github.com/klauspost/compress/tree/master/zstd#zstd +package zstd + +import ( + "errors" + "log" + "math/bits" +) + +const debug = false +const debugSequences = false +const debugMatches = false + +// force encoder to use predefined tables. +const forcePreDef = false + +// zstdMinMatch is the minimum zstd match length. +const zstdMinMatch = 3 + +var ( + // ErrReservedBlockType is returned when a reserved block type is found. + // Typically this indicates wrong or corrupted input. + ErrReservedBlockType = errors.New("invalid input: reserved block type encountered") + + // ErrCompressedSizeTooBig is returned when a block is bigger than allowed. + // Typically this indicates wrong or corrupted input. + ErrCompressedSizeTooBig = errors.New("invalid input: compressed size too big") + + // ErrBlockTooSmall is returned when a block is too small to be decoded. + // Typically returned on invalid input. + ErrBlockTooSmall = errors.New("block too small") + + // ErrMagicMismatch is returned when a "magic" number isn't what is expected. + // Typically this indicates wrong or corrupted input. + ErrMagicMismatch = errors.New("invalid input: magic number mismatch") + + // ErrWindowSizeExceeded is returned when a reference exceeds the valid window size. + // Typically this indicates wrong or corrupted input. + ErrWindowSizeExceeded = errors.New("window size exceeded") + + // ErrWindowSizeTooSmall is returned when no window size is specified. + // Typically this indicates wrong or corrupted input. + ErrWindowSizeTooSmall = errors.New("invalid input: window size was too small") + + // ErrDecoderSizeExceeded is returned if decompressed size exceeds the configured limit. + ErrDecoderSizeExceeded = errors.New("decompressed size exceeds configured limit") + + // ErrUnknownDictionary is returned if the dictionary ID is unknown. + // For the time being dictionaries are not supported. + ErrUnknownDictionary = errors.New("unknown dictionary") + + // ErrFrameSizeExceeded is returned if the stated frame size is exceeded. + // This is only returned if SingleSegment is specified on the frame. + ErrFrameSizeExceeded = errors.New("frame size exceeded") + + // ErrCRCMismatch is returned if CRC mismatches. + ErrCRCMismatch = errors.New("CRC check failed") + + // ErrDecoderClosed will be returned if the Decoder was used after + // Close has been called. + ErrDecoderClosed = errors.New("decoder used after Close") +) + +func println(a ...interface{}) { + if debug { + log.Println(a...) + } +} + +func printf(format string, a ...interface{}) { + if debug { + log.Printf(format, a...) + } +} + +// matchLen returns the maximum length. +// a must be the shortest of the two. +// The function also returns whether all bytes matched. +func matchLen(a, b []byte) int { + b = b[:len(a)] + for i := 0; i < len(a)-7; i += 8 { + if diff := load64(a, i) ^ load64(b, i); diff != 0 { + return i + (bits.TrailingZeros64(diff) >> 3) + } + } + checked := (len(a) >> 3) << 3 + a = a[checked:] + b = b[checked:] + // TODO: We could do a 4 check. + for i := range a { + if a[i] != b[i] { + return int(i) + checked + } + } + return len(a) + checked +} + +// matchLen returns a match length in src between index s and t +func matchLenIn(src []byte, s, t int32) int32 { + s1 := len(src) + b := src[t:] + a := src[s:s1] + b = b[:len(a)] + // Extend the match to be as long as possible. + for i := range a { + if a[i] != b[i] { + return int32(i) + } + } + return int32(len(a)) +} + +func load3232(b []byte, i int32) uint32 { + // Help the compiler eliminate bounds checks on the read so it can be done in a single read. + b = b[i:] + b = b[:4] + return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 +} + +func load6432(b []byte, i int32) uint64 { + // Help the compiler eliminate bounds checks on the read so it can be done in a single read. + b = b[i:] + b = b[:8] + return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | + uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 +} + +func load64(b []byte, i int) uint64 { + // Help the compiler eliminate bounds checks on the read so it can be done in a single read. + b = b[i:] + b = b[:8] + return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | + uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 +} diff --git a/vendor/github.com/moby/locker/LICENSE b/vendor/github.com/moby/locker/LICENSE new file mode 100644 index 000000000..2e0ec1dcf --- /dev/null +++ b/vendor/github.com/moby/locker/LICENSE @@ -0,0 +1,190 @@ + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2013-2018 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/moby/locker/README.md b/vendor/github.com/moby/locker/README.md new file mode 100644 index 000000000..a0852f0f8 --- /dev/null +++ b/vendor/github.com/moby/locker/README.md @@ -0,0 +1,65 @@ +Locker +===== + +locker provides a mechanism for creating finer-grained locking to help +free up more global locks to handle other tasks. + +The implementation looks close to a sync.Mutex, however, the user must provide a +reference to use to refer to the underlying lock when locking and unlocking, +and unlock may generate an error. + +If a lock with a given name does not exist when `Lock` is called, one is +created. +Lock references are automatically cleaned up on `Unlock` if nothing else is +waiting for the lock. + + +## Usage + +```go +package important + +import ( + "sync" + "time" + + "github.com/moby/locker" +) + +type important struct { + locks *locker.Locker + data map[string]interface{} + mu sync.Mutex +} + +func (i *important) Get(name string) interface{} { + i.locks.Lock(name) + defer i.locks.Unlock(name) + return i.data[name] +} + +func (i *important) Create(name string, data interface{}) { + i.locks.Lock(name) + defer i.locks.Unlock(name) + + i.createImportant(data) + + i.mu.Lock() + i.data[name] = data + i.mu.Unlock() +} + +func (i *important) createImportant(data interface{}) { + time.Sleep(10 * time.Second) +} +``` + +For functions dealing with a given name, always lock at the beginning of the +function (or before doing anything with the underlying state), this ensures any +other function that is dealing with the same name will block. + +When needing to modify the underlying data, use the global lock to ensure nothing +else is modifying it at the same time. +Since name lock is already in place, no reads will occur while the modification +is being performed. + diff --git a/vendor/github.com/moby/locker/locker.go b/vendor/github.com/moby/locker/locker.go new file mode 100644 index 000000000..0b22ddfab --- /dev/null +++ b/vendor/github.com/moby/locker/locker.go @@ -0,0 +1,112 @@ +/* +Package locker provides a mechanism for creating finer-grained locking to help +free up more global locks to handle other tasks. + +The implementation looks close to a sync.Mutex, however the user must provide a +reference to use to refer to the underlying lock when locking and unlocking, +and unlock may generate an error. + +If a lock with a given name does not exist when `Lock` is called, one is +created. +Lock references are automatically cleaned up on `Unlock` if nothing else is +waiting for the lock. +*/ +package locker + +import ( + "errors" + "sync" + "sync/atomic" +) + +// ErrNoSuchLock is returned when the requested lock does not exist +var ErrNoSuchLock = errors.New("no such lock") + +// Locker provides a locking mechanism based on the passed in reference name +type Locker struct { + mu sync.Mutex + locks map[string]*lockCtr +} + +// lockCtr is used by Locker to represent a lock with a given name. +type lockCtr struct { + mu sync.Mutex + // waiters is the number of waiters waiting to acquire the lock + // this is int32 instead of uint32 so we can add `-1` in `dec()` + waiters int32 +} + +// inc increments the number of waiters waiting for the lock +func (l *lockCtr) inc() { + atomic.AddInt32(&l.waiters, 1) +} + +// dec decrements the number of waiters waiting on the lock +func (l *lockCtr) dec() { + atomic.AddInt32(&l.waiters, -1) +} + +// count gets the current number of waiters +func (l *lockCtr) count() int32 { + return atomic.LoadInt32(&l.waiters) +} + +// Lock locks the mutex +func (l *lockCtr) Lock() { + l.mu.Lock() +} + +// Unlock unlocks the mutex +func (l *lockCtr) Unlock() { + l.mu.Unlock() +} + +// New creates a new Locker +func New() *Locker { + return &Locker{ + locks: make(map[string]*lockCtr), + } +} + +// Lock locks a mutex with the given name. If it doesn't exist, one is created +func (l *Locker) Lock(name string) { + l.mu.Lock() + if l.locks == nil { + l.locks = make(map[string]*lockCtr) + } + + nameLock, exists := l.locks[name] + if !exists { + nameLock = &lockCtr{} + l.locks[name] = nameLock + } + + // increment the nameLock waiters while inside the main mutex + // this makes sure that the lock isn't deleted if `Lock` and `Unlock` are called concurrently + nameLock.inc() + l.mu.Unlock() + + // Lock the nameLock outside the main mutex so we don't block other operations + // once locked then we can decrement the number of waiters for this lock + nameLock.Lock() + nameLock.dec() +} + +// Unlock unlocks the mutex with the given name +// If the given lock is not being waited on by any other callers, it is deleted +func (l *Locker) Unlock(name string) error { + l.mu.Lock() + nameLock, exists := l.locks[name] + if !exists { + l.mu.Unlock() + return ErrNoSuchLock + } + + if nameLock.count() == 0 { + delete(l.locks, name) + } + nameLock.Unlock() + + l.mu.Unlock() + return nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 23ce53de6..09cfffa28 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -23,26 +23,9 @@ github.com/Masterminds/sprig/v3 # github.com/Masterminds/squirrel v1.5.3 => github.com/Masterminds/squirrel v0.0.0-20161115235646-20f192218cf5 ## explicit github.com/Masterminds/squirrel -# github.com/Microsoft/go-winio v0.5.1 => github.com/Microsoft/go-winio v0.4.12 +# github.com/Microsoft/go-winio v0.5.2 => github.com/Microsoft/go-winio v0.4.12 ## explicit github.com/Microsoft/go-winio -# github.com/Microsoft/hcsshim v0.9.1 => github.com/Microsoft/hcsshim v0.8.6 -## explicit -github.com/Microsoft/hcsshim -github.com/Microsoft/hcsshim/internal/guestrequest -github.com/Microsoft/hcsshim/internal/guid -github.com/Microsoft/hcsshim/internal/hcs -github.com/Microsoft/hcsshim/internal/hcserror -github.com/Microsoft/hcsshim/internal/hns -github.com/Microsoft/hcsshim/internal/interop -github.com/Microsoft/hcsshim/internal/logfields -github.com/Microsoft/hcsshim/internal/longpath -github.com/Microsoft/hcsshim/internal/mergemaps -github.com/Microsoft/hcsshim/internal/safefile -github.com/Microsoft/hcsshim/internal/schema1 -github.com/Microsoft/hcsshim/internal/schema2 -github.com/Microsoft/hcsshim/internal/timeout -github.com/Microsoft/hcsshim/internal/wclayer # github.com/NYTimes/gziphandler v1.1.1 => github.com/NYTimes/gziphandler v1.1.1 ## explicit; go 1.11 github.com/NYTimes/gziphandler @@ -141,8 +124,8 @@ github.com/chai2010/gettext-go github.com/chai2010/gettext-go/mo github.com/chai2010/gettext-go/plural github.com/chai2010/gettext-go/po -# github.com/containerd/containerd v1.6.8 => github.com/containerd/containerd v1.4.13 -## explicit +# github.com/containerd/containerd v1.6.8 => github.com/containerd/containerd v1.5.16 +## explicit; go 1.17 github.com/containerd/containerd/archive/compression github.com/containerd/containerd/content github.com/containerd/containerd/content/local @@ -155,8 +138,9 @@ github.com/containerd/containerd/platforms github.com/containerd/containerd/reference github.com/containerd/containerd/remotes github.com/containerd/containerd/remotes/docker +github.com/containerd/containerd/remotes/docker/auth github.com/containerd/containerd/remotes/docker/schema1 -github.com/containerd/containerd/sys +github.com/containerd/containerd/remotes/errors github.com/containerd/containerd/version # github.com/containernetworking/cni v1.1.2 => github.com/containernetworking/cni v1.1.2 ## explicit; go 1.14 @@ -560,6 +544,13 @@ github.com/jszwec/csvutil # github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e => github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e ## explicit github.com/kevinburke/ssh_config +# github.com/klauspost/compress v1.13.6 => github.com/klauspost/compress v1.9.5 +## explicit +github.com/klauspost/compress/fse +github.com/klauspost/compress/huff0 +github.com/klauspost/compress/snappy +github.com/klauspost/compress/zstd +github.com/klauspost/compress/zstd/internal/xxhash # github.com/konsorten/go-windows-terminal-sequences v1.0.2 => github.com/konsorten/go-windows-terminal-sequences v1.0.2 ## explicit github.com/konsorten/go-windows-terminal-sequences @@ -623,7 +614,7 @@ github.com/mattn/go-isatty # github.com/mattn/go-runewidth v0.0.9 => github.com/mattn/go-runewidth v0.0.4 ## explicit github.com/mattn/go-runewidth -# github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 => github.com/matttproud/golang_protobuf_extensions v1.0.1 +# github.com/matttproud/golang_protobuf_extensions v1.0.4 => github.com/matttproud/golang_protobuf_extensions v1.0.1 ## explicit github.com/matttproud/golang_protobuf_extensions/pbutil # github.com/mitchellh/copystructure v1.2.0 => github.com/mitchellh/copystructure v1.0.0 @@ -641,6 +632,9 @@ github.com/mitchellh/mapstructure # github.com/mitchellh/reflectwalk v1.0.2 => github.com/mitchellh/reflectwalk v1.0.0 ## explicit github.com/mitchellh/reflectwalk +# github.com/moby/locker v1.0.1 => github.com/moby/locker v1.0.1 +## explicit; go 1.13 +github.com/moby/locker # github.com/moby/spdystream v0.2.0 => github.com/moby/spdystream v0.2.0 ## explicit; go 1.13 github.com/moby/spdystream @@ -841,7 +835,7 @@ github.com/patrickmn/go-cache # github.com/pelletier/go-buffruneio v0.2.0 => github.com/pelletier/go-buffruneio v0.2.0 ## explicit github.com/pelletier/go-buffruneio -# github.com/pelletier/go-toml v1.7.0 => github.com/pelletier/go-toml v1.7.0 +# github.com/pelletier/go-toml v1.8.1 => github.com/pelletier/go-toml v1.7.0 ## explicit; go 1.12 github.com/pelletier/go-toml # github.com/peterbourgon/diskv v2.0.1+incompatible => github.com/peterbourgon/diskv v2.0.1+incompatible @@ -1426,7 +1420,7 @@ gopkg.in/inf.v0 # gopkg.in/natefinch/lumberjack.v2 v2.0.0 => gopkg.in/natefinch/lumberjack.v2 v2.0.0 ## explicit gopkg.in/natefinch/lumberjack.v2 -# gopkg.in/square/go-jose.v2 v2.4.0 => gopkg.in/square/go-jose.v2 v2.4.0 +# gopkg.in/square/go-jose.v2 v2.5.1 => gopkg.in/square/go-jose.v2 v2.4.0 ## explicit gopkg.in/square/go-jose.v2 gopkg.in/square/go-jose.v2/cipher @@ -2602,6 +2596,7 @@ sigs.k8s.io/yaml # github.com/alecthomas/template => github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 # github.com/alecthomas/units => github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d # github.com/alessio/shellescape => github.com/alessio/shellescape v1.2.2 +# github.com/alexflint/go-filemutex => github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae # github.com/aliyun/aliyun-oss-go-sdk => github.com/aliyun/aliyun-oss-go-sdk v2.0.4+incompatible # github.com/andreyvit/diff => github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 # github.com/andybalholm/cascadia => github.com/andybalholm/cascadia v1.0.0 @@ -2627,6 +2622,7 @@ sigs.k8s.io/yaml # github.com/bgentry/speakeasy => github.com/bgentry/speakeasy v0.1.0 # github.com/bitly/go-hostpool => github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 # github.com/bitly/go-simplejson => github.com/bitly/go-simplejson v0.5.0 +# github.com/bits-and-blooms/bitset => github.com/bits-and-blooms/bitset v1.2.0 # github.com/blang/semver => github.com/blang/semver v3.5.0+incompatible # github.com/blang/semver/v4 => github.com/blang/semver/v4 v4.0.0 # github.com/bmizerany/assert => github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 @@ -2635,6 +2631,7 @@ sigs.k8s.io/yaml # github.com/bradfitz/gomemcache => github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b # github.com/brancz/kube-rbac-proxy => github.com/brancz/kube-rbac-proxy v0.5.0 # github.com/bshuster-repo/logrus-logstash-hook => github.com/bshuster-repo/logrus-logstash-hook v0.4.1 +# github.com/buger/jsonparser => github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44 # github.com/bugsnag/bugsnag-go => github.com/bugsnag/bugsnag-go v1.5.0 # github.com/bugsnag/osext => github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b # github.com/bugsnag/panicwrap => github.com/bugsnag/panicwrap v1.2.0 @@ -2670,15 +2667,28 @@ sigs.k8s.io/yaml # github.com/cockroachdb/datadriven => github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa # github.com/codahale/hdrhistogram => github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd # github.com/codegangsta/cli => github.com/codegangsta/cli v1.20.0 +# github.com/containerd/aufs => github.com/containerd/aufs v1.0.0 +# github.com/containerd/btrfs => github.com/containerd/btrfs v1.0.0 # github.com/containerd/cgroups => github.com/containerd/cgroups v1.0.2 # github.com/containerd/console => github.com/containerd/console v1.0.3 -# github.com/containerd/containerd => github.com/containerd/containerd v1.4.13 +# github.com/containerd/containerd => github.com/containerd/containerd v1.5.16 # github.com/containerd/continuity => github.com/containerd/continuity v0.0.0-20181203112020-004b46473808 +# github.com/containerd/fifo => github.com/containerd/fifo v1.0.0 +# github.com/containerd/go-cni => github.com/containerd/go-cni v1.0.2 +# github.com/containerd/go-runc => github.com/containerd/go-runc v1.0.0 +# github.com/containerd/imgcrypt => github.com/containerd/imgcrypt v1.1.1 +# github.com/containerd/nri => github.com/containerd/nri v0.1.0 # github.com/containerd/stargz-snapshotter/estargz => github.com/containerd/stargz-snapshotter/estargz v0.7.0 +# github.com/containerd/ttrpc => github.com/containerd/ttrpc v1.1.0 +# github.com/containerd/typeurl => github.com/containerd/typeurl v1.0.2 +# github.com/containerd/zfs => github.com/containerd/zfs v1.0.0 # github.com/containernetworking/cni => github.com/containernetworking/cni v1.1.2 +# github.com/containernetworking/plugins => github.com/containernetworking/plugins v0.9.1 +# github.com/containers/ocicrypt => github.com/containers/ocicrypt v1.1.1 # github.com/coreos/bbolt => github.com/coreos/bbolt v1.3.3 # github.com/coreos/etcd => github.com/coreos/etcd v3.3.17+incompatible # github.com/coreos/go-etcd => github.com/coreos/go-etcd v2.0.0+incompatible +# github.com/coreos/go-iptables => github.com/coreos/go-iptables v0.5.0 # github.com/coreos/go-oidc => github.com/coreos/go-oidc v2.1.0+incompatible # github.com/coreos/go-semver => github.com/coreos/go-semver v0.3.0 # github.com/coreos/go-systemd => github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f @@ -2699,6 +2709,10 @@ sigs.k8s.io/yaml # github.com/cznic/sortutil => github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65 # github.com/cznic/strutil => github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186 # github.com/cznic/zappy => github.com/cznic/zappy v0.0.0-20160723133515-2533cb5b45cc +# github.com/d2g/dhcp4 => github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c +# github.com/d2g/dhcp4client => github.com/d2g/dhcp4client v1.0.0 +# github.com/d2g/dhcp4server => github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5 +# github.com/d2g/hardwareaddr => github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4 # github.com/daaku/go.zipexe => github.com/daaku/go.zipexe v1.0.0 # github.com/dave/jennifer => github.com/dave/jennifer v1.2.0 # github.com/davecgh/go-spew => github.com/davecgh/go-spew v1.1.1 @@ -2911,6 +2925,7 @@ sigs.k8s.io/yaml # github.com/influxdata/roaring => github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6 # github.com/influxdata/tdigest => github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9 # github.com/influxdata/usage-client => github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368 +# github.com/j-keck/arping => github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56 # github.com/jackc/fake => github.com/jackc/fake v0.0.0-20150926172116-812a484cc733 # github.com/jackc/pgx => github.com/jackc/pgx v3.2.0+incompatible # github.com/jbenet/go-context => github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 @@ -2988,11 +3003,13 @@ sigs.k8s.io/yaml # github.com/mdlayher/wifi => github.com/mdlayher/wifi v0.0.0-20190303161829-b1436901ddee # github.com/mgutz/ansi => github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b # github.com/miekg/dns => github.com/miekg/dns v1.1.29 +# github.com/miekg/pkcs11 => github.com/miekg/pkcs11 v1.0.3 # github.com/mikefarah/yaml/v2 => github.com/mikefarah/yaml/v2 v2.4.0 # github.com/mikefarah/yq/v2 => github.com/mikefarah/yq/v2 v2.4.1 # github.com/minio/md5-simd => github.com/minio/md5-simd v1.1.0 # github.com/minio/minio-go/v7 => github.com/minio/minio-go/v7 v7.0.2 # github.com/minio/sha256-simd => github.com/minio/sha256-simd v0.1.1 +# github.com/mistifyio/go-zfs => github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible # github.com/mitchellh/cli => github.com/mitchellh/cli v1.0.0 # github.com/mitchellh/copystructure => github.com/mitchellh/copystructure v1.0.0 # github.com/mitchellh/go-homedir => github.com/mitchellh/go-homedir v1.1.0 @@ -3005,6 +3022,7 @@ sigs.k8s.io/yaml # github.com/moby/locker => github.com/moby/locker v1.0.1 # github.com/moby/spdystream => github.com/moby/spdystream v0.2.0 # github.com/moby/sys/mountinfo => github.com/moby/sys/mountinfo v0.5.0 +# github.com/moby/sys/symlink => github.com/moby/sys/symlink v0.1.0 # github.com/moby/term => github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 # github.com/modern-go/concurrent => github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd # github.com/modern-go/reflect2 => github.com/modern-go/reflect2 v1.0.2 @@ -3105,6 +3123,7 @@ sigs.k8s.io/yaml # github.com/russross/blackfriday => github.com/russross/blackfriday v1.5.2 # github.com/russross/blackfriday/v2 => github.com/russross/blackfriday/v2 v2.1.0 # github.com/ryanuber/columnize => github.com/ryanuber/columnize v2.1.0+incompatible +# github.com/safchain/ethtool => github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8 # github.com/samuel/go-zookeeper => github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da # github.com/santhosh-tekuri/jsonschema => github.com/santhosh-tekuri/jsonschema v1.2.4 # github.com/satori/go.uuid => github.com/satori/go.uuid v1.2.0 @@ -3136,12 +3155,14 @@ sigs.k8s.io/yaml # github.com/spf13/pflag => github.com/spf13/pflag v1.0.5 # github.com/spf13/viper => github.com/spf13/viper v1.4.0 # github.com/src-d/gcfg => github.com/src-d/gcfg v1.4.0 +# github.com/stefanberger/go-pkcs11uri => github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980 # github.com/stoewer/go-strcase => github.com/stoewer/go-strcase v1.2.0 # github.com/streadway/amqp => github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271 # github.com/streadway/handy => github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a # github.com/stretchr/objx => github.com/stretchr/objx v0.2.0 # github.com/stretchr/testify => github.com/stretchr/testify v1.4.0 # github.com/syndtr/gocapability => github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 +# github.com/tchap/go-patricia => github.com/tchap/go-patricia v2.2.6+incompatible # github.com/tchap/go-patricia/v2 => github.com/tchap/go-patricia/v2 v2.3.1 # github.com/thanos-io/thanos => github.com/thanos-io/thanos v0.13.1-0.20200910143741-e0b7f7b32e9c # github.com/tidwall/pretty => github.com/tidwall/pretty v1.0.0 @@ -3203,6 +3224,7 @@ sigs.k8s.io/yaml # go.etcd.io/etcd/raft/v3 => go.etcd.io/etcd/raft/v3 v3.5.4 # go.etcd.io/etcd/server/v3 => go.etcd.io/etcd/server/v3 v3.5.4 # go.mongodb.org/mongo-driver => go.mongodb.org/mongo-driver v1.10.4 +# go.mozilla.org/pkcs7 => go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1 # go.opencensus.io => go.opencensus.io v0.22.3 # go.opentelemetry.io/contrib => go.opentelemetry.io/contrib v0.20.0 # go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc => go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 @@ -3296,6 +3318,7 @@ sigs.k8s.io/yaml # k8s.io/code-generator => k8s.io/code-generator v0.25.3 # k8s.io/component-base => k8s.io/component-base v0.25.3 # k8s.io/component-helpers => k8s.io/component-helpers v0.25.3 +# k8s.io/cri-api => k8s.io/cri-api v0.20.6 # k8s.io/gengo => k8s.io/gengo v0.0.0-20211129171323-c02415ce4185 # k8s.io/klog => k8s.io/klog v1.0.0 # k8s.io/klog/v2 => k8s.io/klog/v2 v2.70.1