{"namespace":"osintverse-community","name":"cdx-toolkit","kind":"tool","summary":"Lists a domain's pages in Common Crawl's URL index, with each capture's time, status, type and language","latest_version":"0.9.39","categories":["web-archive","domain-dns"],"inputs":["domain"],"outputs":["url"],"runtime":"pipx","stars":0,"installs_30d":0,"health":"unknown","health_checked_at":null,"verified":true,"signed":true,"needs_keys":false,"manifest_held_by_registry":false,"community_packaged":true,"source_unavailable":false,"status":"active","ethics":{"touches_target":false,"auth_required":false,"collects_personal_data":false,"notifies_target":false},"egress":true,"sample":{"output_format":"jsonl","stdout":"{\"languages\": \"eng\", \"length\": \"996\", \"mime\": \"text/html\", \"status\": \"200\", \"timestamp\": \"20261005112841\", \"url\": \"https://example.com/\"}\n{\"languages\": \"eng\", \"length\": \"999\", \"mime\": \"text/html\", \"status\": \"200\", \"timestamp\": \"20261005151603\", \"url\": \"https://example.com/\"}\n{\"languages\": \"eng\", \"length\": \"993\", \"mime\": \"text/html\", \"status\": \"200\", \"timestamp\": \"20261005154750\", \"url\": \"http://example.com/\"}\n{\"languages\": \"eng\", \"length\": \"995\", \"mime\": \"text/html\", \"status\": \"200\", \"timestamp\": \"20261005185728\", \"url\": \"http://example.com/\"}\n{\"languages\": \"eng\", \"length\": \"998\", \"mime\": \"text/html\", \"status\": \"200\", \"timestamp\": \"20261005220006\", \"url\": \"https://example.com/\"}\n{\"languages\": \"eng\", \"length\": \"999\", \"mime\": \"text/html\", \"status\": \"200\", \"timestamp\": \"20261005231354\", \"url\": \"https://example.com/\"}\n{\"languages\": \"eng\", \"length\": \"993\", \"mime\": \"text/html\", \"status\": \"200\", \"timestamp\": \"20261005235803\", \"url\": \"http://example.com/\"}\n{\"languages\": \"eng\", \"length\": \"996\", \"mime\": \"text/html\", \"status\": \"200\", \"timestamp\": \"20261006030441\", \"url\": \"https://example.com/\"}\n{\"languages\": \"eng\", \"length\": \"1004\", \"mime\": \"text/html\", \"status\": \"200\", \"timestamp\": \"20261006033845\", \"url\": \"https://www.example.com/\"}\n{\"languages\": \"eng\", \"length\": \"998\", \"mime\": \"text/html\", \"status\": \"200\", \"timestamp\": \"20261006082447\", \"url\": \"https://example.com/\"}\n{\"languages\": \"eng\", \"length\": \"1002\", \"mime\": \"text/html\", \"status\": \"200\", \"timestamp\": \"20261006084244\", \"url\": \"https://www.example.com/\"}\n{\"languages\": \"eng\", \"length\": \"1000\", \"mime\": \"text/html\", \"status\": \"200\", \"timestamp\": \"20261006124306\", \"url\": \"https://example.com/\"}\n{\"languages\": \"eng\", \"length\": \"993\", \"mime\": \"text/html\", \"status\": \"200\", \"timestamp\": \"20261006143312\", \"url\": \"http://example.com/\"}\n{\"languages\": \"eng\", \"length\": \"993\", \"mime\": \"text/html\", \"status\": \"200\", \"timestamp\": \"20261006173838\", \"url\": \"http://www.example.com/\"}\n{\"languages\": \"eng\", \"length\": \"995\", \"mime\": \"text/html\",","version":"0.9.39","recorded_at":1792861121293},"repo_url":"https://github.com/osintverse-community/cdx-toolkit","published_via":"tag","homepage":"https://github.com/commoncrawl/cdx_toolkit","license":"Apache-2.0","manifest":{"schema":1,"name":"cdx-toolkit","namespace":"osintverse-community","version":"0.9.39","summary":"Lists a domain's pages in Common Crawl's URL index, with each capture's time, status, type and language","readme":"README.md","license":"Apache-2.0","homepage":"https://github.com/commoncrawl/cdx_toolkit","categories":["web-archive","domain-dns"],"tags":["common-crawl","urls","web-archive","cdx"],"inputs":[{"name":"domain","type":"domain","required":true,"description":"Domain or host name; pages on its subdomains are not included","example":"example.com"},{"name":"crawls","type":"integer","required":false,"default":1,"description":"How many of the latest crawls (about one a month) to search"},{"name":"limit","type":"integer","required":false,"default":100,"description":"Most captures to return"}],"outputs":[{"type":"url"}],"runtime":{"kind":"pipx","package":"cdx-toolkit","package_version":"0.9.39","index":"pypi","python":">=3.9","entrypoint":"cdxt","wheel_sha256":"577ae9a9578f33ce4bae6c887074f0f70331bf4d6a66a26d1e20836b3004e431","command":["cdxt","--crawl","{crawls}","--limit","{limit}","iter","--fields","url,timestamp,status,mime,languages,length","--jsonl","{domain}/*"],"output_format":"jsonl","timeout_seconds":600,"typical_seconds":30},"network":{"egress":true},"ethics":{"touches_target":false,"auth_required":false,"collects_personal_data":false,"notifies_target":false},"test":{"input":{"domain":"example.com","crawls":1,"limit":20}},"kind":"tool"},"community":{"open_broken":0,"open_outdated":0}}