- root: file:///epyc/data3/stevengs/proc_lsst/scrubbed/cache/
- expiration mode: datasets=4
- root: file:///epyc/data3/stevengs/proc_lsst/scrubbed/cache/
- expiration mode: datasets=4
INFO 2023-11-20T18:46:34.175-08:00 lsst.ctrl.mpexec.cmdLineFwk ()(cmdLineFwk.py:881) - QuantumGraph contains 11 quanta for 1 task, graph ID: '1700534311.840023-99104'
Quanta  Tasks  
------ --------
    11 overscan
- root: file:///epyc/data3/stevengs/proc_lsst/scrubbed/cache/
- expiration mode: datasets=4
'OpenSSL':'23.2.0',
'_libgcc_mutex':'0.1',
'_openmp_mutex':'4.5',
'_sysroot_linux-64_curr_repodata_hack':'3',
'abc':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'afw':'g4728e35f0d+53a8438ab4 with boost=unknown eigen=unknown',
'alembic':'1.12.0',
'aom':'3.6.1',
'apr':'1.7.0',
'argcomplete':'3.1.2',
'argparse':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'array':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'asdf':'3.0.0',
'asdf-astropy':'0.4.0',
'asdf-coordinates-schemas':'0.2.0',
'asdf-standard':'1.0.3',
'asdf-transform-schemas':'0.4.0',
'asdf-unit-schemas':'0.1.0',
'asdf-wcs-schemas':'0.1.1',
'asdf_astropy':'0.4.0',
'asdf_standard':'1.0.3',
'ast':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'asteval':'0.9.31',
'astro_metadata_translator':'g0978aabd1f+0ce179faa8',
'astropy':'5.3.4',
'astroquery':'0.4.6',
'asyncio':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'atexit':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'attr':'23.1.0',
'attrs':'23.1.0',
'autograd':'1.6.2',
'aws-c-auth':'0.7.4',
'aws-c-cal':'0.6.7',
'aws-c-common':'0.9.4',
'aws-c-compression':'0.2.17',
'aws-c-event-stream':'0.3.2',
'aws-c-http':'0.7.13',
'aws-c-io':'0.13.35',
'aws-c-mqtt':'0.9.8',
'aws-c-s3':'0.3.19',
'aws-c-sdkutils':'0.1.12',
'aws-checksums':'0.1.17',
'aws-crt-cpp':'0.24.3',
'aws-sam-translator':'1.79.0',
'aws-sdk-cpp':'1.11.182',
'aws-xray-sdk':'2.12.1',
'backoff':'2.2.1',
'backports':'1.0',
'backports.functools_lru_cache':'1.6.5',
'backports.zoneinfo':'0.2.1',
'base':'g98ffbb4407+81bc2a20b4',
'base64':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'bcrypt':'4.0.1',
'beautifulsoup4':'4.12.2',
'binascii':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'binutils':'2.40',
'binutils_impl_linux-64':'2.40',
'binutils_linux-64':'2.40',
'bisect':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'blinker':'1.6.3',
'blosc':'1.21.5',
'boltons':'23.0.0',
'boost':'1.78.0',
'boost-cpp':'1.78.0',
'boto3':'1.28.69',
'botocore':'1.31.69',
'bottleneck':'1.3.7',
'brotli':'1.1.0',
'brotli-bin':'1.1.0',
'brotli-python':'1.1.0',
'brunsli':'0.1',
'builtins':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'bz2':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'bzip2':'1.0.8',
'c-ares':'1.20.1',
'c-blosc2':'2.10.5',
'c-compiler':'1.5.2',
'cProfile':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'ca-certificates':'2023.7.22',
'cached-property':'1.5.2',
'cached_property':'1.5.2',
'cachetools':'5.3.1',
'calendar':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'cassandra-driver':'3.28.0',
'certifi':'2023.7.22',
'cffi':'1.16.0',
'cfitsio':'4.2.0',
'cfn-lint':'0.83.0',
'charls':'2.4.2',
'charset-normalizer':'3.3.1',
'cli_helpers':'2.3.0',
'click':'8.1.7',
'cmake':'3.27.6',
'cmath':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'codecs':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'collections':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'colorama':'0.4.6',
'coloredlogs':'15.0.1',
'colorsys':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'compilers':'1.5.2',
'concurrent':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'conda':'23.9.0',
'conda-package-handling':'2.2.0',
'conda-package-streaming':'0.9.0',
'conda_env':'lsst-scipipe-7.0.1',
'configobj':'5.0.8',
'configparser':'5.3.0',
'contextlib':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'contextvars':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'contourpy':'1.1.1',
'coord':'1.3.0',
'copy':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'copyreg':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'coverage':'7.3.2',
'coveralls':'1.8.0',
'cryptography':'41.0.4',
'csv':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'ctrl_mpexec':'g76ae3ab134+f0199d472f',
'ctypes':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'curl':'8.4.0',
'cxx-compiler':'1.5.2',
'cycler':'0.12.1',
'cython':'0.29.36',
'daf_base':'gf041782ebf+595decdb33 with boost=unknown',
'daf_butler':'ga1d28be6d8+a9e5a04819',
'daf_relation':'g8a20fa5453+e1da24fbab',
'dataclasses':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'datetime':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'dateutil':'2.8.2',
'dav1d':'1.2.1',
'dbus':'1.13.6',
'decimal':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'decorator':'5.1.1',
'deprecated':'1.2.14',
'difflib':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'dill':'0.3.7',
'dis':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'distutils':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'docker-py':'6.1.3',
'docopt':'0.6.2',
'dogpile.cache':'1.2.2',
'doxygen':'1.9.8',
'dustmaps':'1.0.12',
'ecdsa':'0.18.0',
'eigen':'3.4.0',
'email':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'emcee':'3.1.4',
'encodings':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'enum':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'erfa':'2.0.1.1',
'errno':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'esutil':'0.6.11',
'eups':'2.2.5',
'exceptiongroup':'1.1.3',
'execnet':'2.0.2',
'expat':'2.5.0',
'fastavro':'1.8.4',
'fcntl':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'fftw':'3.3.10',
'filelock':'3.12.4',
'firefly-client':'2.9.0',
'fitsio':'1.2.0',
'fitsverify':'4.20',
'flake8':'6.0.0',
'flask':'3.0.0',
'flask_cors':'3.0.10',
'fmt':'9.1.0',
'fnmatch':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'fonttools':'4.43.1',
'fortran-compiler':'1.5.2',
'fractions':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'freetype':'2.12.1',
'frozendict':'2.3.8',
'ftplib':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'functools':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'future':'0.18.3',
'galsim':'2.5.0',
'gc':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'gcc':'11.4.0',
'gcc_impl_linux-64':'11.4.0',
'gcc_linux-64':'11.4.0',
'genericpath':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'geom':'ga1e77700b3+32ccf48d86 with boost=unknown eigen=unknown',
'geomet':'0.2.1.post1',
'getcalspec':'2.1.0',
'getopt':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'getpass':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'gettext':'0.21.1',
'gflags':'2.2.2',
'gfortran':'11.4.0',
'gfortran_impl_linux-64':'11.4.0',
'gfortran_linux-64':'11.4.0',
'giflib':'5.2.1',
'git':'2.42.0',
'git-lfs':'3.4.0',
'glob':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'globus-sdk':'3.29.0',
'glog':'0.6.0',
'gmp':'6.2.1',
'gmpy2':'2.1.2',
'graphql-core':'3.2.3',
'greenlet':'3.0.0',
'grp':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'gsl':'2.7',
'gsoap':'2.8.123',
'gwcs':'0.19.0',
'gxx':'11.4.0',
'gxx_impl_linux-64':'11.4.0',
'gxx_linux-64':'11.4.0',
'gzip':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'h5py':'3.10.0',
'hashlib':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'hdf5':'1.14.2',
'healpy':'1.16.5',
'healsparse':'1.7.0',
'heapq':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'hmac':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'hpgeom':'1.0.0',
'htcondor':'10.8.0',
'htcondor-classads':'10.8.0',
'htcondor-utils':'10.8.0',
'html':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'html5lib':'1.1',
'http':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'humanfriendly':'10.0',
'humanize':'4.8.0',
'hypothesis':'6.88.1',
'icu':'73.2',
'idds-client':'1.3.3',
'idds-common':'1.3.3',
'idds-doma':'1.3.3',
'idds-workflow':'1.3.3',
'idna':'3.4',
'imagecodecs':'2023.9.18',
'imageio':'2.31.5',
'iminuit':'2.24.0',
'importlib':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'importlib-metadata':'6.8.0',
'importlib-resources':'6.1.0',
'importlib_metadata':'6.8.0',
'importlib_resources':'6.1.0',
'iniconfig':'2.0.0',
'inspect':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'io':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'ip_isr':'ga7ab7f6f60+b95567d9ac with boost=unknown eigen=unknown',
'ipaddress':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'itertools':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'itsdangerous':'2.1.2',
'jaraco.classes':'3.3.0',
'jeepney':'0.8.0',
'jinja2':'3.1.2',
'jmespath':'1.0.1',
'joblib':'1.3.2',
'jq':'1.6',
'jschema-to-python':'1.2.3',
'json':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'jsondiff':'2.0.0',
'jsonpatch':'1.33',
'jsonpickle':'3.0.2',
'jsonpointer':'2.4',
'jsonschema':'4.19.1',
'jsonschema-path':'0.3.1',
'jsonschema-specifications':'2023.7.1',
'junit-xml':'1.9',
'jxrlib':'1.1',
'kernel-headers_linux-64':'3.10.0',
'keyring':'24.2.0',
'keyutils':'1.6.1',
'keyword':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'kiwisolver':'1.4.5',
'krb5':'1.21.2',
'lazy-object-proxy':'1.9.0',
'lazy_loader':'0.3',
'lcms2':'2.15',
'ld_impl_linux-64':'2.40',
'lerc':'4.0.0',
'libabseil':'20230802.1',
'libaec':'1.1.2',
'libapr':'1.7.0',
'libapriconv':'1.2.2',
'libaprutil':'1.6.1',
'libarrow':'12.0.1',
'libavif16':'1.0.1',
'libblas':'3.9.0',
'libbrotlicommon':'1.1.0',
'libbrotlidec':'1.1.0',
'libbrotlienc':'1.1.0',
'libcblas':'3.9.0',
'libcondor_utils':'10.8.0',
'libcrc32c':'1.1.2',
'libcurl':'8.4.0',
'libcxx':'16.0.6',
'libcxxabi':'16.0.6',
'libdeflate':'1.19',
'libedit':'3.1.20191231',
'libev':'4.33',
'libevent':'2.1.12',
'libexpat':'2.5.0',
'libffi':'3.4.2',
'libgcc-devel_linux-64':'11.4.0',
'libgcc-ng':'13.2.0',
'libgfortran-ng':'13.2.0',
'libgfortran5':'13.2.0',
'libglib':'2.78.0',
'libgomp':'13.2.0',
'libgoogle-cloud':'2.12.0',
'libgrpc':'1.58.1',
'libhwloc':'2.9.3',
'libiconv':'1.17',
'libjpeg-turbo':'3.0.0',
'liblapack':'3.9.0',
'liblapacke':'3.9.0',
'libllvm14':'14.0.6',
'libnghttp2':'1.52.0',
'libnsl':'2.0.1',
'libnuma':'2.0.16',
'libpng':'1.6.39',
'libpq':'16.0',
'libprotobuf':'4.24.3',
'libre2-11':'2023.06.02',
'libsanitizer':'11.4.0',
'libsodium':'1.0.18',
'libsqlite':'3.43.2',
'libssh2':'1.11.0',
'libstdcxx-devel_linux-64':'11.4.0',
'libstdcxx-ng':'13.2.0',
'libthrift':'0.19.0',
'libtiff':'4.6.0',
'libutf8proc':'2.8.0',
'libuuid':'2.38.1',
'libuv':'1.46.0',
'libwebp-base':'1.3.2',
'libxcb':'1.15',
'libxml2':'2.11.5',
'libxslt':'1.1.37',
'libzlib':'1.2.13',
'libzopfli':'1.0.3',
'linecache':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'llvm-openmp':'17.0.3',
'llvmlite':'0.40.1',
'lmfit':'1.2.2',
'locale':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'log':'g1e7d6db67d+e1da24fbab with boost=unknown',
'log4cxx':'0.11.0',
'logging':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'lsstdesc.coord':'1.3.0',
'lxml':'4.9.3',
'lz4-c':'1.9.4',
'lzma':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'lzo':'2.10',
'make':'4.3',
'mako':'1.2.4',
'markupsafe':'2.1.3',
'marshal':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'math':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'matplotlib-base':'3.8.0',
'mccabe':'0.7.0',
'meas_algorithms':'g7dc46dff8f+10617abd6a with boost=unknown eigen=unknown',
'meas_base':'g159b3db504+62ba8334a5 with boost=unknown eigen=unknown',
'mimetypes':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'minuit2':'6.22.06',
'minuit2_standalone':'6.22.06',
'mkl':'2022.2.1',
'mmap':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'more-itertools':'10.1.0',
'moto':'4.2.6',
'mpc':'1.3.1',
'mpfr':'4.2.1',
'mpi':'1.0',
'mpi4py':'3.1.5',
'mpich':'4.1.2',
'mpmath':'1.3.0',
'multidict':'6.0.4',
'multiprocessing':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'munge':'0.5.13',
'munkres':'1.1.4',
'ncurses':'6.4',
'ndarray':'1.6.4',
'networkx':'3.2',
'nose':'1.3.7',
'ntpath':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'numba':'0.57.1',
'numbers':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'numexpr':'2.8.7',
'numpy':'1.24.4',
'obs_base':'gcc620a9fd9+6540d52e01',
'oniguruma':'6.9.9',
'opcode':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'openapi-schema-validator':'0.6.2',
'openapi-spec-validator':'0.7.1',
'openjpeg':'2.5.0',
'openssl':'3.1.4',
'operator':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'orc':'1.9.0',
'os':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'packaging':'23.2',
'panda-client':'1.5.66',
'pandas':'2.0.3',
'paramiko':'3.3.1',
'parsl':'2023.6.12',
'pathable':'0.4.3',
'pathlib':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'patsy':'0.5.3',
'pbr':'5.11.1',
'pcre2':'10.40',
'pendulum':'2.1.2',
'pep8-naming':'0.13.3',
'perl':'5.32.1',
'pex_config':'g22560d942e+0ce179faa8',
'pex_exceptions':'gdaeeff99f8+0d8dbea60f with boost=unknown',
'pgcli':'3.5.0',
'pgplot':'5.2.2',
'pgspecial':'2.1.0',
'photutils':'1.8.0',
'pickle':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'piff':'1.3.3',
'pillow':'10.1.0',
'pip':'23.3.1',
'pipe_base':'g655761b648+94da4844e8',
'pkgutil':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'pkgutil-resolve-name':'1.3.10',
'platform':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'platformdirs':'3.11.0',
'pluggy':'1.3.0',
'pooch':'1.7.0',
'posix':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'posixpath':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'pprint':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'prmon':'3.0.2',
'profile':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'progressbar2':'4.2.0',
'proj':'9.3.0',
'prompt-toolkit':'3.0.39',
'prompt_toolkit':'3.0.39',
'pstats':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'psutil':'5.9.5',
'psycopg':'3.1.12',
'psycopg-c':'3.1.12',
'psycopg2':'2.9.7',
'pthread-stubs':'0.4',
'pwd':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'py-cpuinfo':'9.0.0',
'pyarrow':'12.0.1',
'pyasn1':'0.5.0',
'pybind11':'2.10.4',
'pybind11-global':'2.10.4',
'pycodestyle':'2.10.0',
'pycosat':'0.6.6',
'pycparser':'2.21',
'pydantic':'1.10.13',
'pydoc':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'pyerfa':'2.0.1.1',
'pyexpat':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'pyflakes':'3.0.1',
'pygments':'2.16.1',
'pyjwt':'2.8.0',
'pyld':'2.0.3',
'pynacl':'1.5.0',
'pyopenssl':'23.2.0',
'pyparsing':'3.1.1',
'pyproj':'3.6.1',
'pysocks':'1.7.1',
'pysynphot':'2.0.0',
'pytables':'3.9.1',
'pytest':'7.4.2',
'pytest-arraydiff':'0.5.0',
'pytest-astropy':'0.11.0',
'pytest-astropy-header':'0.2.2',
'pytest-cov':'4.1.0',
'pytest-doctestplus':'1.0.0',
'pytest-filter-subpackage':'0.1.2',
'pytest-mock':'3.12.0',
'pytest-openfiles':'0.5.0',
'pytest-remotedata':'0.4.1',
'pytest-runner':'6.0.0',
'pytest-session2file':'0.1.11',
'pytest-subtests':'0.11.0',
'pytest-vcr':'1.0.2',
'pytest-xdist':'3.3.1',
'python':'3.11.6',
'python-dateutil':'2.8.2',
'python-htcondor':'10.8.0',
'python-jose':'3.3.0',
'python-tzdata':'2023.3',
'python-utils':'3.8.1',
'python_abi':'3.11',
'pytorch':'2.0.0',
'pytz':'2023.3.post1',
'pytzdata':'2020.1',
'pyvo':'1.4.2',
'pywavelets':'1.4.1',
'pywin32-on-windows':'0.1.0',
'pyyaml':'6.0.1',
'pyzmq':'25.1.1',
'queue':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'quopri':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'random':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'rav1e':'0.6.6',
'rdma-core':'28.9',
're':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
're2':'2023.06.02',
'readline':'8.2',
'referencing':'0.30.2',
'regex':'2023.10.3',
'reprlib':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'requests':'2.31.0',
'resource':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'resources':'g24611a4c00+e1da24fbab',
'responses':'0.23.1',
'rfc3339-validator':'0.1.4',
'rfc3339_validator':'0.1.4',
'rhash':'1.4.4',
'rpds-py':'0.10.6',
'rsa':'4.9',
'ruamel.yaml':'0.17.40',
'ruamel.yaml.clib':'0.2.7',
'rubin-env':'7.0.1',
'rubin-env-nosysroot':'7.0.1',
'ruff':'0.0.278',
'runpy':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
's2n':'1.3.55',
's3transfer':'0.7.0',
'sarif-om':'1.0.4',
'schwimmbad':'0.3.2',
'scikit-image':'0.22.0',
'scikit-learn':'1.3.2',
'scipy':'1.10.1',
'scitokens-cpp':'1.0.2',
'scons':'4.5.2',
'seaborn':'0.13.0',
'seaborn-base':'0.13.0',
'secrets':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'secretstorage':'3.3.3',
'select':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'selectors':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'semantic_version':'2.10.0',
'setproctitle':'1.3.3',
'setuptools':'68.2.2',
'setuptools-scm':'8.0.4',
'shlex':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'shutil':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'signal':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'site':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'six':'1.16.0',
'skymap':'g93973518b8+01a0ac5232 with boost=unknown eigen=unknown',
'skyproj':'1.1.0',
'sleef':'3.5.1',
'snappy':'1.1.10',
'socket':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'sortedcontainers':'2.4.0',
'soupsieve':'2.5',
'spdlog':'1.11.0',
'spherematch':'0.10.1',
'sphgeom':'g6ae5381d9b+81bc2a20b4',
'sqlalchemy':'2.0.22',
'sqlite':'3.43.2',
'sqlite3':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'sqlparse':'0.4.4',
'sshpubkeys':'3.3.1',
'ssl':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'starlink-ast':'9.2.10',
'stat':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'statsmodels':'0.14.0',
'stevedore':'5.1.0',
'string':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'stringprep':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'struct':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'subprocess':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'svt-av1':'1.7.0',
'sympy':'1.12',
'sys':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'sysconfig':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'sysroot_linux-64':'2.17',
'tabulate':'0.9.0',
'tarfile':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'tbb':'2021.10.0',
'tblib':'2.0.0',
'tempfile':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'termios':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'textwrap':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'threading':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'threadpoolctl':'3.2.0',
'tifffile':'2023.9.26',
'time':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'timeit':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'tk':'8.6.13',
'token':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'tokenize':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'toml':'0.10.2',
'tomli':'2.0.1',
'tomllib':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'torchvision':'0.15.2',
'tqdm':'4.66.1',
'traceback':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'treecorr':'4.3.3',
'treegp':'0.6.0',
'truststore':'0.8.0',
'typeguard':'2.13.3',
'types':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'types-cryptography':'3.3.23.2',
'types-enum34':'1.1.8',
'types-ipaddress':'1.0.8',
'types-paramiko':'3.3.0.0',
'types-pyyaml':'6.0.12.12',
'types-requests':'2.31.0.6',
'types-six':'1.16.21.9',
'types-urllib3':'1.26.25.14',
'typing':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'typing-extensions':'4.8.0',
'typing_extensions':'4.8.0',
'tzdata':'2023c',
'ucx':'1.15.0',
'uncertainties':'3.1.7',
'unicodedata':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'unittest':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'urllib':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'urllib3':'1.26.18',
'utils':'g3bf5ff22bb+81bc2a20b4',
'uuid':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'vcrpy':'5.1.0',
'voms':'2.1.0rc3',
'warnings':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'wcslib':'7.12',
'wcwidth':'0.2.8',
'weakref':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'webencodings':'0.5.1',
'websocket-client':'1.6.4',
'werkzeug':'3.0.0',
'wheel':'0.41.2',
'wrapt':'1.15.0',
'ws4py':'0.5.1',
'xml':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'xmltodict':'0.13.0',
'xorg-kbproto':'1.0.7',
'xorg-libx11':'1.8.7',
'xorg-libxau':'1.0.11',
'xorg-libxdmcp':'1.1.3',
'xorg-xextproto':'7.3.0',
'xorg-xproto':'7.0.31',
'xpa':'2.1.20',
'xz':'5.2.6',
'yaml':'0.2.5',
'yaml-cpp':'0.7.0',
'yarl':'1.9.2',
'zeromq':'4.3.5',
'zfp':'1.0.0',
'zipfile':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'zipimport':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'zipp':'3.17.0',
'zlib':'1.2.13',
'zlib-ng':'2.0.7',
'zoneinfo':'3.11.6 | packaged by conda-forge | (main, Oct  3 2023, 10:40:35) [GCC 12.3.0]',
'zstandard':'0.21.0',
'zstd':'1.5.5',
})
INFO 2023-11-20T18:46:38.345-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220159, ...})(singleQuantumExecutor.py:204) - Preparing execution of quantum for label=overscan dataId={instrument: 'DECam', detector: 1, exposure: 1220159, ...}.
INFO 2023-11-20T18:46:38.356-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220161, ...})(singleQuantumExecutor.py:204) - Preparing execution of quantum for label=overscan dataId={instrument: 'DECam', detector: 1, exposure: 1220161, ...}.
VERBOSE 2023-11-20T18:46:38.356-08:00 lsst.daf.butler.datastores.fileDatastore (overscan:{instrument: 'DECam', detector: 1, exposure: 1220159, ...})(fileDatastore.py:1594) - Number of datasets found in datastore: 0 out of 3 datasets checked.
INFO 2023-11-20T18:46:38.365-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220156, ...})(singleQuantumExecutor.py:204) - Preparing execution of quantum for label=overscan dataId={instrument: 'DECam', detector: 1, exposure: 1220156, ...}.
VERBOSE 2023-11-20T18:46:38.367-08:00 lsst.daf.butler.datastores.fileDatastore (overscan:{instrument: 'DECam', detector: 1, exposure: 1220161, ...})(fileDatastore.py:1594) - Number of datasets found in datastore: 0 out of 3 datasets checked.
INFO 2023-11-20T18:46:38.370-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220151, ...})(singleQuantumExecutor.py:204) - Preparing execution of quantum for label=overscan dataId={instrument: 'DECam', detector: 1, exposure: 1220151, ...}.
VERBOSE 2023-11-20T18:46:38.373-08:00 lsst.daf.butler.datastores.fileDatastore (overscan:{instrument: 'DECam', detector: 1, exposure: 1220156, ...})(fileDatastore.py:1594) - Number of datasets found in datastore: 0 out of 3 datasets checked.
INFO 2023-11-20T18:46:38.375-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220153, ...})(singleQuantumExecutor.py:204) - Preparing execution of quantum for label=overscan dataId={instrument: 'DECam', detector: 1, exposure: 1220153, ...}.
VERBOSE 2023-11-20T18:46:38.378-08:00 lsst.daf.butler.datastores.fileDatastore (overscan:{instrument: 'DECam', detector: 1, exposure: 1220151, ...})(fileDatastore.py:1594) - Number of datasets found in datastore: 0 out of 3 datasets checked.
VERBOSE 2023-11-20T18:46:38.382-08:00 lsst.daf.butler.datastores.fileDatastore (overscan:{instrument: 'DECam', detector: 1, exposure: 1220153, ...})(fileDatastore.py:1594) - Number of datasets found in datastore: 0 out of 3 datasets checked.
INFO 2023-11-20T18:46:38.408-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220159, ...})(singleQuantumExecutor.py:256) - Constructing task and executing quantum for label=overscan dataId={instrument: 'DECam', detector: 1, exposure: 1220159, ...}.
INFO 2023-11-20T18:46:38.417-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220156, ...})(singleQuantumExecutor.py:256) - Constructing task and executing quantum for label=overscan dataId={instrument: 'DECam', detector: 1, exposure: 1220156, ...}.
INFO 2023-11-20T18:46:38.417-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220161, ...})(singleQuantumExecutor.py:256) - Constructing task and executing quantum for label=overscan dataId={instrument: 'DECam', detector: 1, exposure: 1220161, ...}.
INFO 2023-11-20T18:46:38.420-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220152, ...})(singleQuantumExecutor.py:204) - Preparing execution of quantum for label=overscan dataId={instrument: 'DECam', detector: 1, exposure: 1220152, ...}.
INFO 2023-11-20T18:46:38.422-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220153, ...})(singleQuantumExecutor.py:256) - Constructing task and executing quantum for label=overscan dataId={instrument: 'DECam', detector: 1, exposure: 1220153, ...}.
INFO 2023-11-20T18:46:38.424-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220151, ...})(singleQuantumExecutor.py:256) - Constructing task and executing quantum for label=overscan dataId={instrument: 'DECam', detector: 1, exposure: 1220151, ...}.
VERBOSE 2023-11-20T18:46:38.430-08:00 lsst.daf.butler.datastores.fileDatastore (overscan:{instrument: 'DECam', detector: 1, exposure: 1220152, ...})(fileDatastore.py:1594) - Number of datasets found in datastore: 0 out of 3 datasets checked.
ERROR 2023-11-20T18:46:38.435-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220156, ...})(singleQuantumExecutor.py:266) - Execution of task 'overscan' on quantum {instrument: 'DECam', detector: 1, exposure: 1220156, ...} failed. Exception RuntimeError: Integrity failure in Datastore. Size of file s3://repo/DECam/raw/all/raw/20230709/ct4m20230709t194827/raw_DECam_VR_DECam_c0007_6300_0_2600_0_ct4m20230709t194827_S1_DECam_raw_all.fits.fz (709260480) does not match size recorded in registry of 709583040
ERROR 2023-11-20T18:46:38.436-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220161, ...})(singleQuantumExecutor.py:266) - Execution of task 'overscan' on quantum {instrument: 'DECam', detector: 1, exposure: 1220161, ...} failed. Exception RuntimeError: Integrity failure in Datastore. Size of file s3://repo/DECam/raw/all/raw/20230709/ct4m20230709t195124/raw_DECam_VR_DECam_c0007_6300_0_2600_0_ct4m20230709t195124_S1_DECam_raw_all.fits.fz (709583040) does not match size recorded in registry of 709260480
VERBOSE 2023-11-20T18:46:38.437-08:00 lsst.daf.butler._butler ()(_butler.py:1989) - Ingesting 1 file dataset.
VERBOSE 2023-11-20T18:46:38.437-08:00 lsst.daf.butler._butler ()(_butler.py:2069) - Importing 1 ref of dataset type 'overscan_log' into run 'testing/1'
VERBOSE 2023-11-20T18:46:38.438-08:00 lsst.daf.butler._butler ()(_butler.py:1989) - Ingesting 1 file dataset.
VERBOSE 2023-11-20T18:46:38.438-08:00 lsst.daf.butler._butler ()(_butler.py:2069) - Importing 1 ref of dataset type 'overscan_log' into run 'testing/1'
INFO 2023-11-20T18:46:38.466-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220152, ...})(singleQuantumExecutor.py:256) - Constructing task and executing quantum for label=overscan dataId={instrument: 'DECam', detector: 1, exposure: 1220152, ...}.
INFO 2023-11-20T18:46:38.529-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220154, ...})(singleQuantumExecutor.py:204) - Preparing execution of quantum for label=overscan dataId={instrument: 'DECam', detector: 1, exposure: 1220154, ...}.
VERBOSE 2023-11-20T18:46:38.545-08:00 lsst.daf.butler.datastores.fileDatastore (overscan:{instrument: 'DECam', detector: 1, exposure: 1220154, ...})(fileDatastore.py:1594) - Number of datasets found in datastore: 0 out of 3 datasets checked.
INFO 2023-11-20T18:46:38.594-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220157, ...})(singleQuantumExecutor.py:204) - Preparing execution of quantum for label=overscan dataId={instrument: 'DECam', detector: 1, exposure: 1220157, ...}.
VERBOSE 2023-11-20T18:46:38.606-08:00 lsst.daf.butler.datastores.fileDatastore (overscan:{instrument: 'DECam', detector: 1, exposure: 1220157, ...})(fileDatastore.py:1594) - Number of datasets found in datastore: 0 out of 3 datasets checked.
INFO 2023-11-20T18:46:38.642-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220158, ...})(singleQuantumExecutor.py:204) - Preparing execution of quantum for label=overscan dataId={instrument: 'DECam', detector: 1, exposure: 1220158, ...}.
INFO 2023-11-20T18:46:38.664-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220157, ...})(singleQuantumExecutor.py:256) - Constructing task and executing quantum for label=overscan dataId={instrument: 'DECam', detector: 1, exposure: 1220157, ...}.
VERBOSE 2023-11-20T18:46:38.667-08:00 lsst.daf.butler.datastores.fileDatastore (overscan:{instrument: 'DECam', detector: 1, exposure: 1220158, ...})(fileDatastore.py:1594) - Number of datasets found in datastore: 0 out of 3 datasets checked.
INFO 2023-11-20T18:46:38.697-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220154, ...})(singleQuantumExecutor.py:256) - Constructing task and executing quantum for label=overscan dataId={instrument: 'DECam', detector: 1, exposure: 1220154, ...}.
INFO 2023-11-20T18:46:38.742-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220158, ...})(singleQuantumExecutor.py:256) - Constructing task and executing quantum for label=overscan dataId={instrument: 'DECam', detector: 1, exposure: 1220158, ...}.
INFO 2023-11-20T18:46:38.762-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220160, ...})(singleQuantumExecutor.py:204) - Preparing execution of quantum for label=overscan dataId={instrument: 'DECam', detector: 1, exposure: 1220160, ...}.
VERBOSE 2023-11-20T18:46:38.782-08:00 lsst.daf.butler.datastores.fileDatastore (overscan:{instrument: 'DECam', detector: 1, exposure: 1220160, ...})(fileDatastore.py:1594) - Number of datasets found in datastore: 0 out of 3 datasets checked.
INFO 2023-11-20T18:46:38.845-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220160, ...})(singleQuantumExecutor.py:256) - Constructing task and executing quantum for label=overscan dataId={instrument: 'DECam', detector: 1, exposure: 1220160, ...}.
INFO 2023-11-20T18:46:38.850-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220155, ...})(singleQuantumExecutor.py:204) - Preparing execution of quantum for label=overscan dataId={instrument: 'DECam', detector: 1, exposure: 1220155, ...}.
VERBOSE 2023-11-20T18:46:38.859-08:00 lsst.daf.butler.datastores.fileDatastore (overscan:{instrument: 'DECam', detector: 1, exposure: 1220155, ...})(fileDatastore.py:1594) - Number of datasets found in datastore: 0 out of 3 datasets checked.
INFO 2023-11-20T18:46:38.913-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220155, ...})(singleQuantumExecutor.py:256) - Constructing task and executing quantum for label=overscan dataId={instrument: 'DECam', detector: 1, exposure: 1220155, ...}.
Process task-{instrument: 'DECam', detector: 1, exposure: 1220161, ...}:
Traceback (most recent call last):
  File "/nvme/users/stevengs/opt_lsst/conda/envs/lsst-scipipe-7.0.1/lib/python3.11/multiprocessing/process.py", line 314, in _bootstrap
    self.run()
  File "/nvme/users/stevengs/opt_lsst/conda/envs/lsst-scipipe-7.0.1/lib/python3.11/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/nvme/users/stevengs/opt_lsst/conda/envs/lsst-scipipe-7.0.1/share/eups/Linux64/ctrl_mpexec/g76ae3ab134+f0199d472f/python/lsst/ctrl/mpexec/mpGraphExecutor.py", line 168, in _executeJob
    quantumExecutor.execute(taskDef, quantum)
  File "/nvme/users/stevengs/opt_lsst/conda/envs/lsst-scipipe-7.0.1/share/eups/Linux64/ctrl_mpexec/g76ae3ab134+f0199d472f/python/lsst/ctrl/mpexec/singleQuantumExecutor.py", line 167, in execute
    result = self._execute(taskDef, quantum)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/nvme/users/stevengs/opt_lsst/conda/envs/lsst-scipipe-7.0.1/share/eups/Linux64/ctrl_mpexec/g76ae3ab134+f0199d472f/python/lsst/ctrl/mpexec/singleQuantumExecutor.py", line 264, in _execute
    self.runQuantum(task, quantum, taskDef, limited_butler)
  File "/nvme/users/stevengs/opt_lsst/conda/envs/lsst-scipipe-7.0.1/share/eups/Linux64/ctrl_mpexec/g76ae3ab134+f0199d472f/python/lsst/ctrl/mpexec/singleQuantumExecutor.py", line 466, in runQuantum
    task.runQuantum(butlerQC, inputRefs, outputRefs)
  File "/nvme/users/stevengs/opt_lsst/conda/envs/lsst-scipipe-7.0.1/share/eups/Linux64/ip_isr/ga7ab7f6f60+b95567d9ac/python/lsst/ip/isr/isrTask.py", line 1020, in runQuantum
    inputs = butlerQC.get(inputRefs)
             ^^^^^^^^^^^^^^^^^^^^^^^
  File "/nvme/users/stevengs/opt_lsst/conda/envs/lsst-scipipe-7.0.1/share/eups/Linux64/pipe_base/g655761b648+94da4844e8/python/lsst/pipe/base/_quantumContext.py", line 295, in get
    val = self._get(ref)
          ^^^^^^^^^^^^^^
  File "/nvme/users/stevengs/opt_lsst/conda/envs/lsst-scipipe-7.0.1/share/eups/Linux64/pipe_base/g655761b648+94da4844e8/python/lsst/pipe/base/_quantumContext.py", line 221, in _get
    return self.__butler.get(ref)
           ^^^^^^^^^^^^^^^^^^^^^^
  File "/nvme/users/stevengs/opt_lsst/conda/envs/lsst-scipipe-7.0.1/share/eups/Linux64/daf_butler/ga1d28be6d8+a9e5a04819/python/lsst/daf/butler/_butler.py", line 1428, in get
    return self._datastore.get(ref, parameters=parameters, storageClass=storageClass)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/nvme/users/stevengs/opt_lsst/conda/envs/lsst-scipipe-7.0.1/share/eups/Linux64/daf_butler/ga1d28be6d8+a9e5a04819/python/lsst/daf/butler/datastores/fileDatastore.py", line 2292, in get
    return self._read_artifact_into_memory(getInfo, ref, isComponent=isComponent, cache_ref=cache_ref)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/nvme/users/stevengs/opt_lsst/conda/envs/lsst-scipipe-7.0.1/share/eups/Linux64/daf_butler/ga1d28be6d8+a9e5a04819/python/lsst/daf/butler/datastores/fileDatastore.py", line 1279, in _read_artifact_into_memory
    raise RuntimeError(
RuntimeError: Integrity failure in Datastore. Size of file s3://repo/DECam/raw/all/raw/20230709/ct4m20230709t195124/raw_DECam_VR_DECam_c0007_6300_0_2600_0_ct4m20230709t195124_S1_DECam_raw_all.fits.fz (709583040) does not match size recorded in registry of 709260480
ERROR 2023-11-20T18:46:39.195-08:00 lsst.ctrl.mpexec.mpGraphExecutor ()(mpGraphExecutor.py:606) - Task <TaskDef(lsst.ip.isr.isrTask.IsrTask, label=overscan) dataId={instrument: 'DECam', detector: 1, exposure: 1220161, ...}> failed, exit code=1; processing will continue for remaining tasks.
INFO 2023-11-20T18:46:39.195-08:00 lsst.ctrl.mpexec.mpGraphExecutor ()(mpGraphExecutor.py:656) - Executed 0 quanta successfully, 1 failed and 10 remain out of total 11 quanta.
Process task-{instrument: 'DECam', detector: 1, exposure: 1220156, ...}:
Traceback (most recent call last):
  File "/nvme/users/stevengs/opt_lsst/conda/envs/lsst-scipipe-7.0.1/lib/python3.11/multiprocessing/process.py", line 314, in _bootstrap
    self.run()
  File "/nvme/users/stevengs/opt_lsst/conda/envs/lsst-scipipe-7.0.1/lib/python3.11/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/nvme/users/stevengs/opt_lsst/conda/envs/lsst-scipipe-7.0.1/share/eups/Linux64/ctrl_mpexec/g76ae3ab134+f0199d472f/python/lsst/ctrl/mpexec/mpGraphExecutor.py", line 168, in _executeJob
    quantumExecutor.execute(taskDef, quantum)
  File "/nvme/users/stevengs/opt_lsst/conda/envs/lsst-scipipe-7.0.1/share/eups/Linux64/ctrl_mpexec/g76ae3ab134+f0199d472f/python/lsst/ctrl/mpexec/singleQuantumExecutor.py", line 167, in execute
    result = self._execute(taskDef, quantum)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/nvme/users/stevengs/opt_lsst/conda/envs/lsst-scipipe-7.0.1/share/eups/Linux64/ctrl_mpexec/g76ae3ab134+f0199d472f/python/lsst/ctrl/mpexec/singleQuantumExecutor.py", line 264, in _execute
    self.runQuantum(task, quantum, taskDef, limited_butler)
  File "/nvme/users/stevengs/opt_lsst/conda/envs/lsst-scipipe-7.0.1/share/eups/Linux64/ctrl_mpexec/g76ae3ab134+f0199d472f/python/lsst/ctrl/mpexec/singleQuantumExecutor.py", line 466, in runQuantum
    task.runQuantum(butlerQC, inputRefs, outputRefs)
  File "/nvme/users/stevengs/opt_lsst/conda/envs/lsst-scipipe-7.0.1/share/eups/Linux64/ip_isr/ga7ab7f6f60+b95567d9ac/python/lsst/ip/isr/isrTask.py", line 1020, in runQuantum
    inputs = butlerQC.get(inputRefs)
             ^^^^^^^^^^^^^^^^^^^^^^^
  File "/nvme/users/stevengs/opt_lsst/conda/envs/lsst-scipipe-7.0.1/share/eups/Linux64/pipe_base/g655761b648+94da4844e8/python/lsst/pipe/base/_quantumContext.py", line 295, in get
    val = self._get(ref)
          ^^^^^^^^^^^^^^
  File "/nvme/users/stevengs/opt_lsst/conda/envs/lsst-scipipe-7.0.1/share/eups/Linux64/pipe_base/g655761b648+94da4844e8/python/lsst/pipe/base/_quantumContext.py", line 221, in _get
    return self.__butler.get(ref)
           ^^^^^^^^^^^^^^^^^^^^^^
  File "/nvme/users/stevengs/opt_lsst/conda/envs/lsst-scipipe-7.0.1/share/eups/Linux64/daf_butler/ga1d28be6d8+a9e5a04819/python/lsst/daf/butler/_butler.py", line 1428, in get
    return self._datastore.get(ref, parameters=parameters, storageClass=storageClass)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/nvme/users/stevengs/opt_lsst/conda/envs/lsst-scipipe-7.0.1/share/eups/Linux64/daf_butler/ga1d28be6d8+a9e5a04819/python/lsst/daf/butler/datastores/fileDatastore.py", line 2292, in get
    return self._read_artifact_into_memory(getInfo, ref, isComponent=isComponent, cache_ref=cache_ref)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/nvme/users/stevengs/opt_lsst/conda/envs/lsst-scipipe-7.0.1/share/eups/Linux64/daf_butler/ga1d28be6d8+a9e5a04819/python/lsst/daf/butler/datastores/fileDatastore.py", line 1279, in _read_artifact_into_memory
    raise RuntimeError(
RuntimeError: Integrity failure in Datastore. Size of file s3://repo/DECam/raw/all/raw/20230709/ct4m20230709t194827/raw_DECam_VR_DECam_c0007_6300_0_2600_0_ct4m20230709t194827_S1_DECam_raw_all.fits.fz (709260480) does not match size recorded in registry of 709583040
ERROR 2023-11-20T18:46:39.400-08:00 lsst.ctrl.mpexec.mpGraphExecutor ()(mpGraphExecutor.py:606) - Task <TaskDef(lsst.ip.isr.isrTask.IsrTask, label=overscan) dataId={instrument: 'DECam', detector: 1, exposure: 1220156, ...}> failed, exit code=1; processing will continue for remaining tasks.
INFO 2023-11-20T18:46:39.401-08:00 lsst.ctrl.mpexec.mpGraphExecutor ()(mpGraphExecutor.py:656) - Executed 0 quanta successfully, 2 failed and 9 remain out of total 11 quanta.
INFO 2023-11-20T18:46:49.115-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220159, ...})(isrTask.py:1378) - Converting exposure to floating point values.
INFO 2023-11-20T18:46:49.635-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220160, ...})(isrTask.py:1378) - Converting exposure to floating point values.
INFO 2023-11-20T18:46:50.002-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220158, ...})(isrTask.py:1378) - Converting exposure to floating point values.
INFO 2023-11-20T18:46:50.071-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220159, ...})(isrTask.py:1525) - Widening saturation trails.
INFO 2023-11-20T18:46:50.310-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220154, ...})(isrTask.py:1378) - Converting exposure to floating point values.
INFO 2023-11-20T18:46:50.426-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220158, ...})(isrTask.py:1525) - Widening saturation trails.
INFO 2023-11-20T18:46:50.954-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220160, ...})(isrTask.py:1525) - Widening saturation trails.
WARNING 2023-11-20T18:46:51.087-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220159, ...})(isrTask.py:2572) - No rough magnitude zero point defined for filter VR DECam c0007 6300.0 2600.0.
INFO 2023-11-20T18:46:51.090-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220159, ...})(isrTask.py:2580) - Setting rough magnitude zero point for filter VR DECam c0007 6300.0 2600.0: 30.500000
INFO 2023-11-20T18:46:51.092-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220159, ...})(isrTask.py:1701) - Measuring background level.
WARNING 2023-11-20T18:46:51.122-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220158, ...})(isrTask.py:2572) - No rough magnitude zero point defined for filter VR DECam c0007 6300.0 2600.0.
INFO 2023-11-20T18:46:51.126-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220158, ...})(isrTask.py:2580) - Setting rough magnitude zero point for filter VR DECam c0007 6300.0 2600.0: 30.500000
INFO 2023-11-20T18:46:51.127-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220158, ...})(isrTask.py:1701) - Measuring background level.
INFO 2023-11-20T18:46:51.226-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220154, ...})(isrTask.py:1525) - Widening saturation trails.
WARNING 2023-11-20T18:46:52.025-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220160, ...})(isrTask.py:2572) - No rough magnitude zero point defined for filter VR DECam c0007 6300.0 2600.0.
INFO 2023-11-20T18:46:52.027-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220160, ...})(isrTask.py:2580) - Setting rough magnitude zero point for filter VR DECam c0007 6300.0 2600.0: 30.500000
INFO 2023-11-20T18:46:52.028-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220160, ...})(isrTask.py:1701) - Measuring background level.
INFO 2023-11-20T18:46:52.206-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220159, ...})(isrTask.py:2514) - Flattened sky level: 15337.666992 +/- 475.546929.
INFO 2023-11-20T18:46:52.311-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220157, ...})(isrTask.py:1378) - Converting exposure to floating point values.
INFO 2023-11-20T18:46:52.326-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220155, ...})(isrTask.py:1378) - Converting exposure to floating point values.
INFO 2023-11-20T18:46:52.378-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220158, ...})(isrTask.py:2514) - Flattened sky level: 15349.849609 +/- 475.927332.
INFO 2023-11-20T18:46:52.404-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220151, ...})(isrTask.py:1378) - Converting exposure to floating point values.
WARNING 2023-11-20T18:46:52.415-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220154, ...})(isrTask.py:2572) - No rough magnitude zero point defined for filter VR DECam c0007 6300.0 2600.0.
INFO 2023-11-20T18:46:52.416-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220154, ...})(isrTask.py:2580) - Setting rough magnitude zero point for filter VR DECam c0007 6300.0 2600.0: 30.500000
INFO 2023-11-20T18:46:52.416-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220154, ...})(isrTask.py:1701) - Measuring background level.
INFO 2023-11-20T18:46:52.569-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220152, ...})(isrTask.py:1378) - Converting exposure to floating point values.
INFO 2023-11-20T18:46:52.623-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220160, ...})(isrTask.py:2514) - Flattened sky level: 15319.599609 +/- 474.982810.
INFO 2023-11-20T18:46:52.654-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220159, ...})(isrTask.py:2548) - Measuring sky levels in 8x16 grids: 15196.145980.
INFO 2023-11-20T18:46:52.656-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220155, ...})(isrTask.py:1525) - Widening saturation trails.
INFO 2023-11-20T18:46:52.656-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220159, ...})(isrTask.py:2549) - Sky flatness in 8x16 grids - pp: 0.349931 rms: 0.039872.
INFO 2023-11-20T18:46:52.665-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220157, ...})(isrTask.py:1525) - Widening saturation trails.
INFO 2023-11-20T18:46:52.677-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220151, ...})(isrTask.py:1525) - Widening saturation trails.
INFO 2023-11-20T18:46:52.701-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220158, ...})(isrTask.py:2548) - Measuring sky levels in 8x16 grids: 15207.895782.
INFO 2023-11-20T18:46:52.702-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220158, ...})(isrTask.py:2549) - Sky flatness in 8x16 grids - pp: 0.349949 rms: 0.039872.
INFO 2023-11-20T18:46:52.752-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220154, ...})(isrTask.py:2514) - Flattened sky level: 15205.181641 +/- 471.314636.
INFO 2023-11-20T18:46:52.871-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220152, ...})(isrTask.py:1525) - Widening saturation trails.
INFO 2023-11-20T18:46:52.965-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220160, ...})(isrTask.py:2548) - Measuring sky levels in 8x16 grids: 15177.351468.
INFO 2023-11-20T18:46:52.966-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220160, ...})(isrTask.py:2549) - Sky flatness in 8x16 grids - pp: 0.349882 rms: 0.039873.
WARNING 2023-11-20T18:46:52.973-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220157, ...})(isrTask.py:2572) - No rough magnitude zero point defined for filter VR DECam c0007 6300.0 2600.0.
INFO 2023-11-20T18:46:52.974-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220157, ...})(isrTask.py:2580) - Setting rough magnitude zero point for filter VR DECam c0007 6300.0 2600.0: 30.500000
INFO 2023-11-20T18:46:52.974-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220157, ...})(isrTask.py:1701) - Measuring background level.
WARNING 2023-11-20T18:46:52.991-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220155, ...})(isrTask.py:2572) - No rough magnitude zero point defined for filter VR DECam c0007 6300.0 2600.0.
INFO 2023-11-20T18:46:52.992-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220155, ...})(isrTask.py:2580) - Setting rough magnitude zero point for filter VR DECam c0007 6300.0 2600.0: 30.500000
INFO 2023-11-20T18:46:52.992-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220155, ...})(isrTask.py:1701) - Measuring background level.
WARNING 2023-11-20T18:46:53.017-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220151, ...})(isrTask.py:2572) - No rough magnitude zero point defined for filter VR DECam c0007 6300.0 2600.0.
INFO 2023-11-20T18:46:53.018-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220151, ...})(isrTask.py:2580) - Setting rough magnitude zero point for filter VR DECam c0007 6300.0 2600.0: 30.500000
INFO 2023-11-20T18:46:53.018-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220151, ...})(isrTask.py:1701) - Measuring background level.
INFO 2023-11-20T18:46:53.109-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220154, ...})(isrTask.py:2548) - Measuring sky levels in 8x16 grids: 15063.776537.
INFO 2023-11-20T18:46:53.110-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220154, ...})(isrTask.py:2549) - Sky flatness in 8x16 grids - pp: 0.349773 rms: 0.039852.
WARNING 2023-11-20T18:46:53.204-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220152, ...})(isrTask.py:2572) - No rough magnitude zero point defined for filter VR DECam c0007 6300.0 2600.0.
INFO 2023-11-20T18:46:53.204-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220152, ...})(isrTask.py:2580) - Setting rough magnitude zero point for filter VR DECam c0007 6300.0 2600.0: 30.500000
INFO 2023-11-20T18:46:53.204-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220152, ...})(isrTask.py:1701) - Measuring background level.
INFO 2023-11-20T18:46:53.293-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220157, ...})(isrTask.py:2514) - Flattened sky level: 15173.312500 +/- 470.466189.
INFO 2023-11-20T18:46:53.334-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220155, ...})(isrTask.py:2514) - Flattened sky level: 15252.958008 +/- 472.810275.
INFO 2023-11-20T18:46:53.362-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220151, ...})(isrTask.py:2514) - Flattened sky level: 15317.666992 +/- 474.876682.
INFO 2023-11-20T18:46:53.566-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220152, ...})(isrTask.py:2514) - Flattened sky level: 15263.227539 +/- 473.160944.
INFO 2023-11-20T18:46:53.618-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220157, ...})(isrTask.py:2548) - Measuring sky levels in 8x16 grids: 15033.695315.
INFO 2023-11-20T18:46:53.619-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220157, ...})(isrTask.py:2549) - Sky flatness in 8x16 grids - pp: 0.349708 rms: 0.039852.
INFO 2023-11-20T18:46:53.670-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220155, ...})(isrTask.py:2548) - Measuring sky levels in 8x16 grids: 15111.748199.
INFO 2023-11-20T18:46:53.670-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220155, ...})(isrTask.py:2549) - Sky flatness in 8x16 grids - pp: 0.349764 rms: 0.039857.
INFO 2023-11-20T18:46:53.685-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220151, ...})(isrTask.py:2548) - Measuring sky levels in 8x16 grids: 15175.869253.
INFO 2023-11-20T18:46:53.686-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220151, ...})(isrTask.py:2549) - Sky flatness in 8x16 grids - pp: 0.349871 rms: 0.039867.
INFO 2023-11-20T18:46:53.893-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220152, ...})(isrTask.py:2548) - Measuring sky levels in 8x16 grids: 15121.395917.
INFO 2023-11-20T18:46:53.893-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220152, ...})(isrTask.py:2549) - Sky flatness in 8x16 grids - pp: 0.349766 rms: 0.039860.
INFO 2023-11-20T18:46:55.275-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220158, ...})(singleQuantumExecutor.py:279) - Execution of task 'overscan' on quantum {instrument: 'DECam', detector: 1, exposure: 1220158, ...} took 16.639 seconds
VERBOSE 2023-11-20T18:46:55.276-08:00 lsst.daf.butler._butler ()(_butler.py:1989) - Ingesting 1 file dataset.
VERBOSE 2023-11-20T18:46:55.276-08:00 lsst.daf.butler._butler ()(_butler.py:2069) - Importing 1 ref of dataset type 'overscan_log' into run 'testing/1'
INFO 2023-11-20T18:46:56.576-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220153, ...})(isrTask.py:1378) - Converting exposure to floating point values.
INFO 2023-11-20T18:46:56.635-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220157, ...})(singleQuantumExecutor.py:279) - Execution of task 'overscan' on quantum {instrument: 'DECam', detector: 1, exposure: 1220157, ...} took 18.043 seconds
INFO 2023-11-20T18:46:56.635-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220160, ...})(singleQuantumExecutor.py:279) - Execution of task 'overscan' on quantum {instrument: 'DECam', detector: 1, exposure: 1220160, ...} took 17.880 seconds
VERBOSE 2023-11-20T18:46:56.636-08:00 lsst.daf.butler._butler ()(_butler.py:1989) - Ingesting 1 file dataset.
VERBOSE 2023-11-20T18:46:56.636-08:00 lsst.daf.butler._butler ()(_butler.py:2069) - Importing 1 ref of dataset type 'overscan_log' into run 'testing/1'
VERBOSE 2023-11-20T18:46:56.637-08:00 lsst.daf.butler._butler ()(_butler.py:1989) - Ingesting 1 file dataset.
VERBOSE 2023-11-20T18:46:56.638-08:00 lsst.daf.butler._butler ()(_butler.py:2069) - Importing 1 ref of dataset type 'overscan_log' into run 'testing/1'
INFO 2023-11-20T18:46:56.664-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220151, ...})(singleQuantumExecutor.py:279) - Execution of task 'overscan' on quantum {instrument: 'DECam', detector: 1, exposure: 1220151, ...} took 18.296 seconds
VERBOSE 2023-11-20T18:46:56.665-08:00 lsst.daf.butler._butler ()(_butler.py:1989) - Ingesting 1 file dataset.
VERBOSE 2023-11-20T18:46:56.665-08:00 lsst.daf.butler._butler ()(_butler.py:2069) - Importing 1 ref of dataset type 'overscan_log' into run 'testing/1'
INFO 2023-11-20T18:46:56.670-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220159, ...})(singleQuantumExecutor.py:279) - Execution of task 'overscan' on quantum {instrument: 'DECam', detector: 1, exposure: 1220159, ...} took 18.328 seconds
VERBOSE 2023-11-20T18:46:56.672-08:00 lsst.daf.butler._butler ()(_butler.py:1989) - Ingesting 1 file dataset.
VERBOSE 2023-11-20T18:46:56.672-08:00 lsst.daf.butler._butler ()(_butler.py:2069) - Importing 1 ref of dataset type 'overscan_log' into run 'testing/1'
INFO 2023-11-20T18:46:56.831-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220152, ...})(singleQuantumExecutor.py:279) - Execution of task 'overscan' on quantum {instrument: 'DECam', detector: 1, exposure: 1220152, ...} took 18.414 seconds
VERBOSE 2023-11-20T18:46:56.833-08:00 lsst.daf.butler._butler ()(_butler.py:1989) - Ingesting 1 file dataset.
VERBOSE 2023-11-20T18:46:56.833-08:00 lsst.daf.butler._butler ()(_butler.py:2069) - Importing 1 ref of dataset type 'overscan_log' into run 'testing/1'
INFO 2023-11-20T18:46:56.837-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220153, ...})(isrTask.py:1525) - Widening saturation trails.
INFO 2023-11-20T18:46:56.863-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220154, ...})(singleQuantumExecutor.py:279) - Execution of task 'overscan' on quantum {instrument: 'DECam', detector: 1, exposure: 1220154, ...} took 18.337 seconds
VERBOSE 2023-11-20T18:46:56.865-08:00 lsst.daf.butler._butler ()(_butler.py:1989) - Ingesting 1 file dataset.
VERBOSE 2023-11-20T18:46:56.865-08:00 lsst.daf.butler._butler ()(_butler.py:2069) - Importing 1 ref of dataset type 'overscan_log' into run 'testing/1'
INFO 2023-11-20T18:46:56.866-08:00 lsst.ctrl.mpexec.mpGraphExecutor ()(mpGraphExecutor.py:656) - Executed 3 quanta successfully, 2 failed and 6 remain out of total 11 quanta.
INFO 2023-11-20T18:46:56.967-08:00 lsst.ctrl.mpexec.mpGraphExecutor ()(mpGraphExecutor.py:656) - Executed 4 quanta successfully, 2 failed and 5 remain out of total 11 quanta.
INFO 2023-11-20T18:46:56.967-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220155, ...})(singleQuantumExecutor.py:279) - Execution of task 'overscan' on quantum {instrument: 'DECam', detector: 1, exposure: 1220155, ...} took 18.120 seconds
VERBOSE 2023-11-20T18:46:56.969-08:00 lsst.daf.butler._butler ()(_butler.py:1989) - Ingesting 1 file dataset.
VERBOSE 2023-11-20T18:46:56.970-08:00 lsst.daf.butler._butler ()(_butler.py:2069) - Importing 1 ref of dataset type 'overscan_log' into run 'testing/1'
INFO 2023-11-20T18:46:57.068-08:00 lsst.ctrl.mpexec.mpGraphExecutor ()(mpGraphExecutor.py:656) - Executed 5 quanta successfully, 2 failed and 4 remain out of total 11 quanta.
INFO 2023-11-20T18:46:57.170-08:00 lsst.ctrl.mpexec.mpGraphExecutor ()(mpGraphExecutor.py:656) - Executed 6 quanta successfully, 2 failed and 3 remain out of total 11 quanta.
WARNING 2023-11-20T18:46:57.177-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220153, ...})(isrTask.py:2572) - No rough magnitude zero point defined for filter VR DECam c0007 6300.0 2600.0.
INFO 2023-11-20T18:46:57.177-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220153, ...})(isrTask.py:2580) - Setting rough magnitude zero point for filter VR DECam c0007 6300.0 2600.0: 30.500000
INFO 2023-11-20T18:46:57.178-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220153, ...})(isrTask.py:1701) - Measuring background level.
INFO 2023-11-20T18:46:57.271-08:00 lsst.ctrl.mpexec.mpGraphExecutor ()(mpGraphExecutor.py:656) - Executed 7 quanta successfully, 2 failed and 2 remain out of total 11 quanta.
INFO 2023-11-20T18:46:57.578-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220153, ...})(isrTask.py:2514) - Flattened sky level: 15251.583008 +/- 472.830620.
INFO 2023-11-20T18:46:57.923-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220153, ...})(isrTask.py:2548) - Measuring sky levels in 8x16 grids: 15109.975808.
INFO 2023-11-20T18:46:57.924-08:00 lsst.overscan (overscan:{instrument: 'DECam', detector: 1, exposure: 1220153, ...})(isrTask.py:2549) - Sky flatness in 8x16 grids - pp: 0.349752 rms: 0.039859.
INFO 2023-11-20T18:46:58.282-08:00 lsst.ctrl.mpexec.mpGraphExecutor ()(mpGraphExecutor.py:656) - Executed 8 quanta successfully, 2 failed and 1 remain out of total 11 quanta.
INFO 2023-11-20T18:47:00.522-08:00 lsst.ctrl.mpexec.singleQuantumExecutor (overscan:{instrument: 'DECam', detector: 1, exposure: 1220153, ...})(singleQuantumExecutor.py:279) - Execution of task 'overscan' on quantum {instrument: 'DECam', detector: 1, exposure: 1220153, ...} took 22.149 seconds
VERBOSE 2023-11-20T18:47:00.524-08:00 lsst.daf.butler._butler ()(_butler.py:1989) - Ingesting 1 file dataset.
VERBOSE 2023-11-20T18:47:00.524-08:00 lsst.daf.butler._butler ()(_butler.py:2069) - Importing 1 ref of dataset type 'overscan_log' into run 'testing/1'
INFO 2023-11-20T18:47:00.792-08:00 lsst.ctrl.mpexec.mpGraphExecutor ()(mpGraphExecutor.py:656) - Executed 9 quanta successfully, 2 failed and 0 remain out of total 11 quanta.
ERROR 2023-11-20T18:47:00.793-08:00 lsst.ctrl.mpexec.mpGraphExecutor ()(mpGraphExecutor.py:672) - Failed jobs:
ERROR 2023-11-20T18:47:00.793-08:00 lsst.ctrl.mpexec.mpGraphExecutor ()(mpGraphExecutor.py:675) -   - FAILED: <TaskDef(lsst.ip.isr.isrTask.IsrTask, label=overscan) dataId={instrument: 'DECam', detector: 1, exposure: 1220161, ...}>
ERROR 2023-11-20T18:47:00.793-08:00 lsst.ctrl.mpexec.mpGraphExecutor ()(mpGraphExecutor.py:675) -   - FAILED: <TaskDef(lsst.ip.isr.isrTask.IsrTask, label=overscan) dataId={instrument: 'DECam', detector: 1, exposure: 1220156, ...}>
ERROR 2023-11-20T18:47:00.794-08:00 lsst.daf.butler.cli.utils ()(utils.py:1133) - Caught an exception, details are in traceback:
Traceback (most recent call last):
  File "/nvme/users/stevengs/opt_lsst/conda/envs/lsst-scipipe-7.0.1/share/eups/Linux64/ctrl_mpexec/g76ae3ab134+f0199d472f/python/lsst/ctrl/mpexec/cli/cmd/commands.py", line 219, in run
    script.run(qgraphObj=qgraph, **kwargs)
  File "/nvme/users/stevengs/opt_lsst/conda/envs/lsst-scipipe-7.0.1/share/eups/Linux64/ctrl_mpexec/g76ae3ab134+f0199d472f/python/lsst/ctrl/mpexec/cli/script/run.py", line 225, in run
    f.runPipeline(qgraphObj, taskFactory, args)
  File "/nvme/users/stevengs/opt_lsst/conda/envs/lsst-scipipe-7.0.1/share/eups/Linux64/ctrl_mpexec/g76ae3ab134+f0199d472f/python/lsst/ctrl/mpexec/cmdLineFwk.py", line 827, in runPipeline
    executor.execute(graph)
  File "/nvme/users/stevengs/opt_lsst/conda/envs/lsst-scipipe-7.0.1/share/eups/Linux64/ctrl_mpexec/g76ae3ab134+f0199d472f/python/lsst/ctrl/mpexec/mpGraphExecutor.py", line 409, in execute
    self._executeQuantaMP(graph, self.report)
  File "/nvme/users/stevengs/opt_lsst/conda/envs/lsst-scipipe-7.0.1/share/eups/Linux64/ctrl_mpexec/g76ae3ab134+f0199d472f/python/lsst/ctrl/mpexec/mpGraphExecutor.py", line 681, in _executeQuantaMP
    raise MPGraphExecutorError("One or more tasks failed or timed out during execution.")
lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError: One or more tasks failed or timed out during execution.