pax_global_header00006660000000000000000000000064142141253270014513gustar00rootroot0000000000000052 comment=5407681c34ac299ad5c0622ea0c2d186a330fec1 waybackpy-3.0.6/000077500000000000000000000000001421412532700135135ustar00rootroot00000000000000waybackpy-3.0.6/.github/000077500000000000000000000000001421412532700150535ustar00rootroot00000000000000waybackpy-3.0.6/.github/ISSUE_TEMPLATE/000077500000000000000000000000001421412532700172365ustar00rootroot00000000000000waybackpy-3.0.6/.github/ISSUE_TEMPLATE/bug_report.md000066400000000000000000000011641421412532700217320ustar00rootroot00000000000000--- name: Bug report about: Create a report to help us improve title: '' labels: bug assignees: akamhy --- **Describe the bug** A clear and concise description of what the bug is. **To Reproduce** Steps to reproduce the behavior: 1. Go to '...' 2. Click on '....' 3. Scroll down to '....' 4. See error **Expected behavior** A clear and concise description of what you expected to happen. **Screenshots** If applicable, add screenshots to help explain your problem. **Version:** - OS: [e.g. iOS] - Version [e.g. 22] - Is latest version? [e.g. Yes/No] **Additional context** Add any other context about the problem here. waybackpy-3.0.6/.github/ISSUE_TEMPLATE/feature_request.md000066400000000000000000000011371421412532700227650ustar00rootroot00000000000000--- name: Feature request about: Suggest an idea for this project title: '' labels: enhancement assignees: akamhy --- **Is your feature request related to a problem? Please describe.** A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] **Describe the solution you'd like** A clear and concise description of what you want to happen. **Describe alternatives you've considered** A clear and concise description of any alternative solutions or features you've considered. **Additional context** Add any other context or screenshots about the feature request here. waybackpy-3.0.6/.github/workflows/000077500000000000000000000000001421412532700171105ustar00rootroot00000000000000waybackpy-3.0.6/.github/workflows/build-test.yml000066400000000000000000000015201421412532700217050ustar00rootroot00000000000000# This workflow will install Python dependencies, run tests and lint with a variety of Python versions # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions name: Build on: push: branches: [ master ] pull_request: branches: [ master ] jobs: build: runs-on: ubuntu-latest strategy: matrix: python-version: ['3.7', '3.10'] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip pip install -U setuptools wheel - name: Build test the package run: | python setup.py sdist bdist_wheel waybackpy-3.0.6/.github/workflows/codeql-analysis.yml000066400000000000000000000044411421412532700227260ustar00rootroot00000000000000# For most projects, this workflow file will not need changing; you simply need # to commit it to your repository. # # You may wish to alter this file to override the set of languages analyzed, # or to provide custom queries or build logic. # # ******** NOTE ******** # We have attempted to detect the languages in your repository. Please check # the `language` matrix defined below to confirm you have the correct set of # supported CodeQL languages. # name: "CodeQL" on: push: branches: [ master ] pull_request: # The branches below must be a subset of the branches above branches: [ master ] schedule: - cron: '30 6 * * 1' jobs: analyze: name: Analyze runs-on: ubuntu-latest permissions: actions: read contents: read security-events: write strategy: fail-fast: false matrix: language: [ 'python' ] # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] # Learn more about CodeQL language support at https://git.io/codeql-language-support steps: - name: Checkout repository uses: actions/checkout@v2 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v1 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. # queries: ./path/to/local/query, your-org/your-repo/queries@main # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild uses: github/codeql-action/autobuild@v1 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines # and modify them (or add more) to build your code if your project # uses a compiled language #- run: | # make bootstrap # make release - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v1 waybackpy-3.0.6/.github/workflows/python-publish.yml000066400000000000000000000015411421412532700226210ustar00rootroot00000000000000# This workflows will upload a Python Package using Twine when a release is created # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries name: Upload Python Package on: release: types: [created] jobs: deploy: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v2 with: python-version: '3.x' - name: Install dependencies run: | python -m pip install --upgrade pip pip install setuptools wheel twine - name: Build and publish env: TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | python setup.py sdist bdist_wheel twine upload dist/* waybackpy-3.0.6/.github/workflows/unit-test.yml000066400000000000000000000022371421412532700215730ustar00rootroot00000000000000# This workflow will install Python dependencies, run tests and lint with a variety of Python versions # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions name: Tests on: push: branches: [ master ] pull_request: branches: [ master ] jobs: build: runs-on: ubuntu-latest strategy: matrix: python-version: ['3.10'] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip pip install '.[dev]' - name: Lint with flake8 run: | flake8 . --count --show-source --statistics - name: Lint with black run: | black . --check --diff - name: Static type test with mypy run: | mypy -p waybackpy -p tests - name: Test with pytest run: | pytest - name: Upload coverage to Codecov run: | bash <(curl -s https://codecov.io/bash) -t ${{ secrets.CODECOV_TOKEN }} waybackpy-3.0.6/.gitignore000066400000000000000000000034651421412532700155130ustar00rootroot00000000000000# Files generated while testing *-urls-*.txt # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ pip-wheel-metadata/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # PEP 582; used by e.g. github.com/David-OConnor/pyflow __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ waybackpy-3.0.6/.pep8speaks.yml000066400000000000000000000001561421412532700164010ustar00rootroot00000000000000scanner: diff_only: True linter: flake8 flake8: max-line-length: 88 extend-ignore: W503,W605 waybackpy-3.0.6/.whitesource000066400000000000000000000003261421412532700160560ustar00rootroot00000000000000{ "scanSettings": { "baseBranches": [] }, "checkRunSettings": { "vulnerableCheckRunConclusionLevel": "failure", "displayMode": "diff" }, "issueSettings": { "minSeverityLevel": "LOW" } } waybackpy-3.0.6/CITATION.cff000066400000000000000000000013461421412532700154110ustar00rootroot00000000000000cff-version: 1.2.0 message: "If you use this software, please cite it as below." title: waybackpy abstract: "Python package that interfaces with the Internet Archive's Wayback Machine APIs. Archive pages and retrieve archived pages easily." version: '3.0.6' doi: 10.5281/ZENODO.3977276 date-released: 2022-03-15 type: software authors: - given-names: Akash family-names: Mahanty email: akamhy@yahoo.com orcid: https://orcid.org/0000-0003-2482-8227 keywords: - Archive Website - Wayback Machine - Internet Archive - Wayback Machine CLI - Wayback Machine Python - Internet Archiving - Availability API - CDX API - savepagenow license: MIT repository-code: "https://github.com/akamhy/waybackpy" waybackpy-3.0.6/CODE_OF_CONDUCT.md000066400000000000000000000121501421412532700163110ustar00rootroot00000000000000# Contributor Covenant Code of Conduct ## Our Pledge We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. ## Our Standards Examples of behavior that contributes to a positive environment for our community include: * Demonstrating empathy and kindness toward other people * Being respectful of differing opinions, viewpoints, and experiences * Giving and gracefully accepting constructive feedback * Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience * Focusing on what is best not just for us as individuals, but for the overall community Examples of unacceptable behavior include: * The use of sexualized language or imagery, and sexual attention or advances of any kind * Trolling, insulting or derogatory comments, and personal or political attacks * Public or private harassment * Publishing others' private information, such as a physical or email address, without their explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting ## Enforcement Responsibilities Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate. ## Scope This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at akamhy@yahoo.com. All complaints will be reviewed and investigated promptly and fairly. All community leaders are obligated to respect the privacy and security of the reporter of any incident. ## Enforcement Guidelines Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct: ### 1. Correction **Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. **Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested. ### 2. Warning **Community Impact**: A violation through a single incident or series of actions. **Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban. ### 3. Temporary Ban **Community Impact**: A serious violation of community standards, including sustained inappropriate behavior. **Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. ### 4. Permanent Ban **Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. **Consequence**: A permanent ban from any sort of public interaction within the community. ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.0, available at . Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity). [homepage]: https://www.contributor-covenant.org For answers to common questions about this code of conduct, see the FAQ at . Translations are available at . waybackpy-3.0.6/CONTRIBUTORS.md000066400000000000000000000012071421412532700157720ustar00rootroot00000000000000# CONTRIBUTORS ## AUTHORS - akamhy () - eggplants () - danvalen1 () - AntiCompositeNumber () - rafaelrdealmeida () - jonasjancarik () - jfinkhaeuser () ## ACKNOWLEDGEMENTS - mhmdiaa () for . known_urls is based on this gist. - dequeued0 () for reporting bugs and useful feature requests. waybackpy-3.0.6/LICENSE000066400000000000000000000021701421412532700145200ustar00rootroot00000000000000MIT License Copyright (c) 2020-2022 waybackpy contributors ( https://github.com/akamhy/waybackpy/graphs/contributors ) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. waybackpy-3.0.6/README.md000066400000000000000000000174521421412532700150030ustar00rootroot00000000000000

A Python package & CLI tool that interfaces with the Wayback Machine API

Unit Tests codecov pypi Downloads Codacy Badge GitHub lastest commit PyPI - Python Version Code style: black

--- # Introduction Waybackpy is a Python package and a CLI tool that interfaces with the Wayback Machine APIs. Wayback Machine has 3 client side APIs. - SavePageNow or Save API - CDX Server API - Availability API These three APIs can be accessed via the waybackpy either by importing it from a python file/module or from the command-line interface. ## Installation **Using [pip](https://en.wikipedia.org/wiki/Pip_(package_manager)), from [PyPI](https://pypi.org/) (recommended)**: ```bash pip install waybackpy ``` **Using [conda](https://en.wikipedia.org/wiki/Conda_(package_manager)), from [conda-forge](https://anaconda.org/conda-forge/waybackpy) (recommended)**: See also [waybackpy feedstock](https://github.com/conda-forge/waybackpy-feedstock), maintainers are [@rafaelrdealmeida](https://github.com/rafaelrdealmeida/), [@labriunesp](https://github.com/labriunesp/) and [@akamhy](https://github.com/akamhy/). ```bash conda install -c conda-forge waybackpy ``` **Install directly from [this git repository](https://github.com/akamhy/waybackpy) (NOT recommended)**: ```bash pip install git+https://github.com/akamhy/waybackpy.git ``` ## Docker Image Docker Hub: [hub.docker.com/r/secsi/waybackpy](https://hub.docker.com/r/secsi/waybackpy) Docker image is automatically updated on every release by [Regulary and Automatically Updated Docker Images](https://github.com/cybersecsi/RAUDI) (RAUDI). RAUDI is a tool by [SecSI](https://secsi.io), an Italian cybersecurity startup. ## Usage ### As a Python package #### Save API aka SavePageNow ```python >>> from waybackpy import WaybackMachineSaveAPI >>> url = "https://github.com" >>> user_agent = "Mozilla/5.0 (Windows NT 5.1; rv:40.0) Gecko/20100101 Firefox/40.0" >>> >>> save_api = WaybackMachineSaveAPI(url, user_agent) >>> save_api.save() https://web.archive.org/web/20220118125249/https://github.com/ >>> save_api.cached_save False >>> save_api.timestamp() datetime.datetime(2022, 1, 18, 12, 52, 49) ``` #### CDX API aka CDXServerAPI ```python >>> from waybackpy import WaybackMachineCDXServerAPI >>> url = "https://google.com" >>> user_agent = "my new app's user agent" >>> cdx_api = WaybackMachineCDXServerAPI(url, user_agent) ``` ##### oldest ```python >>> cdx_api.oldest() com,google)/ 19981111184551 http://google.com:80/ text/html 200 HOQ2TGPYAEQJPNUA6M4SMZ3NGQRBXDZ3 381 >>> oldest = cdx_api.oldest() >>> oldest com,google)/ 19981111184551 http://google.com:80/ text/html 200 HOQ2TGPYAEQJPNUA6M4SMZ3NGQRBXDZ3 381 >>> oldest.archive_url 'https://web.archive.org/web/19981111184551/http://google.com:80/' >>> oldest.original 'http://google.com:80/' >>> oldest.urlkey 'com,google)/' >>> oldest.timestamp '19981111184551' >>> oldest.datetime_timestamp datetime.datetime(1998, 11, 11, 18, 45, 51) >>> oldest.statuscode '200' >>> oldest.mimetype 'text/html' ``` ##### newest ```python >>> newest = cdx_api.newest() >>> newest com,google)/ 20220217234427 http://@google.com/ text/html 301 Y6PVK4XWOI3BXQEXM5WLLWU5JKUVNSFZ 563 >>> newest.archive_url 'https://web.archive.org/web/20220217234427/http://@google.com/' >>> newest.timestamp '20220217234427' ``` ##### near ```python >>> near = cdx_api.near(year=2010, month=10, day=10, hour=10, minute=10) >>> near.archive_url 'https://web.archive.org/web/20101010101435/http://google.com/' >>> near com,google)/ 20101010101435 http://google.com/ text/html 301 Y6PVK4XWOI3BXQEXM5WLLWU5JKUVNSFZ 391 >>> near.timestamp '20101010101435' >>> near.timestamp '20101010101435' >>> near = cdx_api.near(wayback_machine_timestamp=2008080808) >>> near.archive_url 'https://web.archive.org/web/20080808051143/http://google.com/' >>> near = cdx_api.near(unix_timestamp=1286705410) >>> near com,google)/ 20101010101435 http://google.com/ text/html 301 Y6PVK4XWOI3BXQEXM5WLLWU5JKUVNSFZ 391 >>> near.archive_url 'https://web.archive.org/web/20101010101435/http://google.com/' >>> ``` ##### snapshots ```python >>> from waybackpy import WaybackMachineCDXServerAPI >>> url = "https://pypi.org" >>> user_agent = "Mozilla/5.0 (Windows NT 5.1; rv:40.0) Gecko/20100101 Firefox/40.0" >>> cdx = WaybackMachineCDXServerAPI(url, user_agent, start_timestamp=2016, end_timestamp=2017) >>> for item in cdx.snapshots(): ... print(item.archive_url) ... https://web.archive.org/web/20160110011047/http://pypi.org/ https://web.archive.org/web/20160305104847/http://pypi.org/ . . # URLS REDACTED FOR READABILITY . https://web.archive.org/web/20171127171549/https://pypi.org/ https://web.archive.org/web/20171206002737/http://pypi.org:80/ ``` #### Availability API It is recommended to not use the availability API due to performance issues. All the methods of availability API interface class, `WaybackMachineAvailabilityAPI`, are also implemented in the CDX server API interface class, `WaybackMachineCDXServerAPI`. Also note that the `newest()` method of `WaybackMachineAvailabilityAPI` can be more recent than `WaybackMachineCDXServerAPI`'s same method. ```python >>> from waybackpy import WaybackMachineAvailabilityAPI >>> >>> url = "https://google.com" >>> user_agent = "Mozilla/5.0 (Windows NT 5.1; rv:40.0) Gecko/20100101 Firefox/40.0" >>> >>> availability_api = WaybackMachineAvailabilityAPI(url, user_agent) ``` ##### oldest ```python >>> availability_api.oldest() https://web.archive.org/web/19981111184551/http://google.com:80/ ``` ##### newest ```python >>> availability_api.newest() https://web.archive.org/web/20220118150444/https://www.google.com/ ``` ##### near ```python >>> availability_api.near(year=2010, month=10, day=10, hour=10) https://web.archive.org/web/20101010101708/http://www.google.com/ ``` > Documentation is at . ### As a CLI tool Demo video on [asciinema.org](https://asciinema.org/a/469890), you can copy the text from video: [![asciicast](https://asciinema.org/a/469890.svg)](https://asciinema.org/a/469890) > CLI documentation is at . waybackpy-3.0.6/_config.yml000066400000000000000000000000331421412532700156360ustar00rootroot00000000000000theme: jekyll-theme-cayman waybackpy-3.0.6/assets/000077500000000000000000000000001421412532700150155ustar00rootroot00000000000000waybackpy-3.0.6/assets/waybackpy_logo.svg000066400000000000000000000205021421412532700205470ustar00rootroot00000000000000 waybackpy-3.0.6/pyproject.toml000066400000000000000000000001321421412532700164230ustar00rootroot00000000000000[build-system] requires = ["wheel", "setuptools"] build-backend = "setuptools.build_meta" waybackpy-3.0.6/requirements-dev.txt000066400000000000000000000001351421412532700175520ustar00rootroot00000000000000black click codecov flake8 mypy pytest pytest-cov requests setuptools>=46.4.0 types-requests waybackpy-3.0.6/requirements.txt000066400000000000000000000000271421412532700167760ustar00rootroot00000000000000click requests urllib3 waybackpy-3.0.6/setup.cfg000066400000000000000000000044061421412532700153400ustar00rootroot00000000000000[metadata] name = waybackpy version = attr: waybackpy.__version__ description = Python package that interfaces with the Internet Archive's Wayback Machine APIs. Archive pages and retrieve archived pages easily. long_description = file: README.md long_description_content_type = text/markdown license = MIT author = Akash Mahanty author_email = akamhy@yahoo.com url = https://akamhy.github.io/waybackpy/ download_url = https://github.com/akamhy/waybackpy/releases project_urls = Documentation = https://github.com/akamhy/waybackpy/wiki Source = https://github.com/akamhy/waybackpy Tracker = https://github.com/akamhy/waybackpy/issues keywords = Archive Website Wayback Machine Internet Archive Wayback Machine CLI Wayback Machine Python Internet Archiving Availability API CDX API savepagenow classifiers = Development Status :: 5 - Production/Stable Intended Audience :: Developers Intended Audience :: End Users/Desktop Natural Language :: English Typing :: Typed License :: OSI Approved :: MIT License Programming Language :: Python Programming Language :: Python :: 3 Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: Implementation :: CPython [options] packages = find: include-package-data = True python_requires = >= 3.6 install_requires = click requests urllib3 [options.package_data] waybackpy = py.typed [options.extras_require] dev = black codecov flake8 mypy pytest pytest-cov setuptools>=46.4.0 types-requests [options.entry_points] console_scripts = waybackpy = waybackpy.cli:main [isort] profile = black [flake8] indent-size = 4 max-line-length = 88 extend-ignore = W503,W605 exclude = venv __pycache__ .venv ./env venv/ env .env ./build [mypy] python_version = 3.9 show_error_codes = True pretty = True strict = True [tool:pytest] addopts = # show summary of all tests that did not pass -ra # enable all warnings -Wd # coverage and html report --cov=waybackpy --cov-report=html testpaths = tests waybackpy-3.0.6/setup.py000066400000000000000000000000461421412532700152250ustar00rootroot00000000000000from setuptools import setup setup() waybackpy-3.0.6/snapcraft.yaml000066400000000000000000000012041421412532700163550ustar00rootroot00000000000000name: waybackpy summary: Wayback Machine API interface and a command-line tool description: | Waybackpy is a CLI tool that interfaces with the Wayback Machine APIs. Wayback Machine has three client side public APIs, Save API, Availability API and CDX API. These three APIs can be accessed via the waybackpy from the terminal. version: git grade: stable confinement: strict base: core20 architectures: - build-on: [arm64, armhf, amd64] apps: waybackpy: command: bin/waybackpy plugs: [home, network, network-bind, removable-media] parts: waybackpy: plugin: python source: https://github.com/akamhy/waybackpy.git waybackpy-3.0.6/tests/000077500000000000000000000000001421412532700146555ustar00rootroot00000000000000waybackpy-3.0.6/tests/__init__.py000066400000000000000000000000001421412532700167540ustar00rootroot00000000000000waybackpy-3.0.6/tests/test_availability_api.py000066400000000000000000000073641421412532700216030ustar00rootroot00000000000000import random import string from datetime import datetime, timedelta import pytest from waybackpy.availability_api import WaybackMachineAvailabilityAPI from waybackpy.exceptions import ( ArchiveNotInAvailabilityAPIResponse, InvalidJSONInAvailabilityAPIResponse, ) now = datetime.utcnow() url = "https://example.com/" user_agent = ( "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36" ) def rndstr(n: int) -> str: return "".join( random.choice(string.ascii_uppercase + string.digits) for _ in range(n) ) def test_oldest() -> None: """ Test the oldest archive of Google.com and also checks the attributes. """ url = "https://example.com/" user_agent = ( "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36" ) availability_api = WaybackMachineAvailabilityAPI(url, user_agent) oldest = availability_api.oldest() oldest_archive_url = oldest.archive_url assert "2002" in oldest_archive_url oldest_timestamp = oldest.timestamp() assert abs(oldest_timestamp - now) > timedelta(days=7000) # More than 19 years assert ( availability_api.json is not None and availability_api.json["archived_snapshots"]["closest"]["available"] is True ) assert repr(oldest).find("example.com") != -1 assert "2002" in str(oldest) def test_newest() -> None: """ Assuming that the recent most Google Archive was made no more earlier than last one day which is 86400 seconds. """ url = "https://www.youtube.com/" user_agent = "Mozilla/5.0 (X11; Linux x86_64; rv:96.0) Gecko/20100101 Firefox/96.0" availability_api = WaybackMachineAvailabilityAPI(url, user_agent) newest = availability_api.newest() newest_timestamp = newest.timestamp() # betting in favor that latest youtube archive was not before the last 3 days # high tarffic sites like youtube are archived mnay times a day, so seems # very reasonable to me. assert abs(newest_timestamp - now) < timedelta(seconds=86400 * 3) def test_invalid_json() -> None: """ When the API is malfunctioning or we don't pass a URL, it may return invalid JSON data. """ with pytest.raises(InvalidJSONInAvailabilityAPIResponse): availability_api = WaybackMachineAvailabilityAPI(url="", user_agent=user_agent) _ = availability_api.archive_url def test_no_archive() -> None: """ ArchiveNotInAvailabilityAPIResponse may be raised if Wayback Machine did not replied with the archive despite the fact that we know the site has million of archives. Don't know the reason for this wierd behavior. And also if really there are no archives for the passed URL this exception is raised. """ with pytest.raises(ArchiveNotInAvailabilityAPIResponse): availability_api = WaybackMachineAvailabilityAPI( url=f"https://{rndstr(30)}.cn", user_agent=user_agent ) _ = availability_api.archive_url def test_no_api_call_str_repr() -> None: """ Some entitled users maybe want to see what is the string representation if they don’t make any API requests. str() must not return None so we return "" """ availability_api = WaybackMachineAvailabilityAPI( url=f"https://{rndstr(30)}.gov", user_agent=user_agent ) assert str(availability_api) == "" def test_no_call_timestamp() -> None: """ If no API requests were made the bound timestamp() method returns the datetime.max as a default value. """ availability_api = WaybackMachineAvailabilityAPI( url=f"https://{rndstr(30)}.in", user_agent=user_agent ) assert datetime.max == availability_api.timestamp() waybackpy-3.0.6/tests/test_cdx_api.py000066400000000000000000000120531421412532700176760ustar00rootroot00000000000000import random import string import pytest from waybackpy.cdx_api import WaybackMachineCDXServerAPI from waybackpy.exceptions import NoCDXRecordFound def rndstr(n: int) -> str: return "".join( random.choice(string.ascii_uppercase + string.digits) for _ in range(n) ) def test_a() -> None: user_agent = ( "Mozilla/5.0 (MacBook Air; M1 Mac OS X 11_4) AppleWebKit/605.1.15 " "(KHTML, like Gecko) Version/14.1.1 Safari/604.1" ) url = "https://twitter.com/jack" wayback = WaybackMachineCDXServerAPI( url=url, user_agent=user_agent, match_type="prefix", collapses=["urlkey"], start_timestamp="201001", end_timestamp="201002", ) # timeframe bound prefix matching enabled along with active urlkey based collapsing snapshots = wayback.snapshots() # for snapshot in snapshots: assert snapshot.timestamp.startswith("2010") def test_b() -> None: user_agent = ( "Mozilla/5.0 (MacBook Air; M1 Mac OS X 11_4) " "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/604.1" ) url = "https://www.google.com" wayback = WaybackMachineCDXServerAPI( url=url, user_agent=user_agent, start_timestamp="202101", end_timestamp="202112", collapses=["urlkey"], ) # timeframe bound prefix matching enabled along with active urlkey based collapsing snapshots = wayback.snapshots() # for snapshot in snapshots: assert snapshot.timestamp.startswith("2021") def test_c() -> None: user_agent = ( "Mozilla/5.0 (MacBook Air; M1 Mac OS X 11_4) " "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/604.1" ) url = "https://www.google.com" cdx = WaybackMachineCDXServerAPI( url=url, user_agent=user_agent, closest="201010101010", sort="closest", limit="1", ) snapshots = cdx.snapshots() for snapshot in snapshots: archive_url = snapshot.archive_url timestamp = snapshot.timestamp break assert str(archive_url).find("google.com") assert "20101010" in timestamp def test_d() -> None: user_agent = ( "Mozilla/5.0 (MacBook Air; M1 Mac OS X 11_4) " "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/604.1" ) cdx = WaybackMachineCDXServerAPI( url="akamhy.github.io", user_agent=user_agent, match_type="prefix", use_pagination=True, filters=["statuscode:200"], ) snapshots = cdx.snapshots() count = 0 for snapshot in snapshots: count += 1 assert str(snapshot.archive_url).find("akamhy.github.io") assert count > 50 def test_oldest() -> None: user_agent = ( "Mozilla/5.0 (MacBook Air; M1 Mac OS X 11_4) " "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/604.1" ) cdx = WaybackMachineCDXServerAPI( url="google.com", user_agent=user_agent, filters=["statuscode:200"], ) oldest = cdx.oldest() assert "1998" in oldest.timestamp assert "google" in oldest.urlkey assert oldest.original.find("google.com") != -1 assert oldest.archive_url.find("google.com") != -1 def test_newest() -> None: user_agent = ( "Mozilla/5.0 (MacBook Air; M1 Mac OS X 11_4) " "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/604.1" ) cdx = WaybackMachineCDXServerAPI( url="google.com", user_agent=user_agent, filters=["statuscode:200"], ) newest = cdx.newest() assert "google" in newest.urlkey assert newest.original.find("google.com") != -1 assert newest.archive_url.find("google.com") != -1 def test_near() -> None: user_agent = ( "Mozilla/5.0 (MacBook Air; M1 Mac OS X 11_4) " "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/604.1" ) cdx = WaybackMachineCDXServerAPI( url="google.com", user_agent=user_agent, filters=["statuscode:200"], ) near = cdx.near(year=2010, month=10, day=10, hour=10, minute=10) assert "2010101010" in near.timestamp assert "google" in near.urlkey assert near.original.find("google.com") != -1 assert near.archive_url.find("google.com") != -1 near = cdx.near(wayback_machine_timestamp="201010101010") assert "2010101010" in near.timestamp assert "google" in near.urlkey assert near.original.find("google.com") != -1 assert near.archive_url.find("google.com") != -1 near = cdx.near(unix_timestamp=1286705410) assert "2010101010" in near.timestamp assert "google" in near.urlkey assert near.original.find("google.com") != -1 assert near.archive_url.find("google.com") != -1 with pytest.raises(NoCDXRecordFound): dne_url = f"https://{rndstr(30)}.in" cdx = WaybackMachineCDXServerAPI( url=dne_url, user_agent=user_agent, filters=["statuscode:200"], ) cdx.near(unix_timestamp=1286705410) waybackpy-3.0.6/tests/test_cdx_snapshot.py000066400000000000000000000025551421412532700207720ustar00rootroot00000000000000from datetime import datetime from waybackpy.cdx_snapshot import CDXSnapshot def test_CDXSnapshot() -> None: sample_input = ( "org,archive)/ 20080126045828 http://github.com " "text/html 200 Q4YULN754FHV2U6Q5JUT6Q2P57WEWNNY 1415" ) prop_values = sample_input.split(" ") properties = {} ( properties["urlkey"], properties["timestamp"], properties["original"], properties["mimetype"], properties["statuscode"], properties["digest"], properties["length"], ) = prop_values snapshot = CDXSnapshot(properties) assert properties["urlkey"] == snapshot.urlkey assert properties["timestamp"] == snapshot.timestamp assert properties["original"] == snapshot.original assert properties["mimetype"] == snapshot.mimetype assert properties["statuscode"] == snapshot.statuscode assert properties["digest"] == snapshot.digest assert properties["length"] == snapshot.length assert ( datetime.strptime(properties["timestamp"], "%Y%m%d%H%M%S") == snapshot.datetime_timestamp ) archive_url = ( "https://web.archive.org/web/" + properties["timestamp"] + "/" + properties["original"] ) assert archive_url == snapshot.archive_url assert sample_input == str(snapshot) assert sample_input == repr(snapshot) waybackpy-3.0.6/tests/test_cdx_utils.py000066400000000000000000000057201421412532700202700ustar00rootroot00000000000000from typing import Any, Dict, List import pytest from waybackpy.cdx_utils import ( check_collapses, check_filters, check_match_type, check_sort, full_url, get_response, get_total_pages, ) from waybackpy.exceptions import WaybackError def test_get_total_pages() -> None: url = "twitter.com" user_agent = ( "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 " "(KHTML, like Gecko) Version/14.0.2 Safari/605.1.15" ) assert get_total_pages(url=url, user_agent=user_agent) >= 56 def test_full_url() -> None: endpoint = "https://web.archive.org/cdx/search/cdx" params: Dict[str, Any] = {} assert endpoint == full_url(endpoint, params) params = {"a": "1"} assert full_url(endpoint, params) == "https://web.archive.org/cdx/search/cdx?a=1" assert ( full_url(endpoint + "?", params) == "https://web.archive.org/cdx/search/cdx?a=1" ) params["b"] = 2 assert ( full_url(endpoint + "?", params) == "https://web.archive.org/cdx/search/cdx?a=1&b=2" ) params["c"] = "foo bar" assert ( full_url(endpoint + "?", params) == "https://web.archive.org/cdx/search/cdx?a=1&b=2&c=foo%20bar" ) def test_get_response() -> None: url = "https://github.com" user_agent = ( "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:78.0) Gecko/20100101 Firefox/78.0" ) headers = {"User-Agent": str(user_agent)} response = get_response(url, headers=headers) assert not isinstance(response, Exception) and response.status_code == 200 def test_check_filters() -> None: filters: List[str] = [] check_filters(filters) filters = ["statuscode:200", "timestamp:20215678901234", "original:https://url.com"] check_filters(filters) with pytest.raises(WaybackError): check_filters("not-list") # type: ignore[arg-type] with pytest.raises(WaybackError): check_filters(["invalid"]) def test_check_collapses() -> None: collapses: List[str] = [] check_collapses(collapses) collapses = ["timestamp:10"] check_collapses(collapses) collapses = ["urlkey"] check_collapses(collapses) collapses = "urlkey" # type: ignore[assignment] with pytest.raises(WaybackError): check_collapses(collapses) collapses = ["also illegal collapse"] with pytest.raises(WaybackError): check_collapses(collapses) def test_check_match_type() -> None: assert check_match_type(None, "url") match_type = "exact" url = "test_url" assert check_match_type(match_type, url) url = "has * in it" with pytest.raises(WaybackError): check_match_type("domain", url) with pytest.raises(WaybackError): check_match_type("not a valid type", "url") def test_check_sort() -> None: assert check_sort("default") assert check_sort("closest") assert check_sort("reverse") with pytest.raises(WaybackError): assert check_sort("random crap") waybackpy-3.0.6/tests/test_cli.py000066400000000000000000000074171421412532700170460ustar00rootroot00000000000000import requests from click.testing import CliRunner from waybackpy import __version__ from waybackpy.cli import main def test_oldest() -> None: runner = CliRunner() result = runner.invoke(main, ["--url", " https://github.com ", "--oldest"]) assert result.exit_code == 0 assert ( result.output == "Archive URL:\nhttps://web.archive.org/web/2008051421\ 0148/http://github.com/\n" ) def test_near() -> None: runner = CliRunner() result = runner.invoke( main, [ "--url", " https://facebook.com ", "--near", "--year", "2010", "--month", "5", "--day", "10", "--hour", "6", ], ) assert result.exit_code == 0 assert ( result.output == "Archive URL:\nhttps://web.archive.org/web/2010051008\ 2647/http://www.facebook.com/\n" ) def test_newest() -> None: runner = CliRunner() result = runner.invoke(main, ["--url", " https://microsoft.com ", "--newest"]) assert result.exit_code == 0 assert ( result.output.find("microsoft.com") != -1 and result.output.find("Archive URL:\n") != -1 ) def test_cdx() -> None: runner = CliRunner() result = runner.invoke( main, "--url https://twitter.com/jack --cdx --user-agent some-user-agent \ --start-timestamp 2010 --end-timestamp 2012 --collapse urlkey \ --match-type prefix --cdx-print archiveurl --cdx-print length \ --cdx-print digest --cdx-print statuscode --cdx-print mimetype \ --cdx-print original --cdx-print timestamp --cdx-print urlkey".split( " " ), ) assert result.exit_code == 0 assert result.output.count("\n") > 3000 def test_save() -> None: runner = CliRunner() result = runner.invoke( main, "--url https://yahoo.com --user_agent my-unique-user-agent \ --save --headers".split( " " ), ) assert result.exit_code == 0 assert result.output.find("Archive URL:") != -1 assert (result.output.find("Cached save:\nTrue") != -1) or ( result.output.find("Cached save:\nFalse") != -1 ) assert result.output.find("Save API headers:\n") != -1 assert result.output.find("yahoo.com") != -1 def test_version() -> None: runner = CliRunner() result = runner.invoke(main, ["--version"]) assert result.exit_code == 0 assert result.output == f"waybackpy version {__version__}\n" def test_license() -> None: runner = CliRunner() result = runner.invoke(main, ["--license"]) assert result.exit_code == 0 assert ( result.output == requests.get( url="https://raw.githubusercontent.com/akamhy/waybackpy/master/LICENSE" ).text + "\n" ) def test_only_url() -> None: runner = CliRunner() result = runner.invoke(main, ["--url", "https://google.com"]) assert result.exit_code == 0 assert ( result.output == "NoCommandFound: Only URL passed, but did not specify what to do with the URL. Use \ --help flag for help using waybackpy.\n" ) def test_known_url() -> None: # with file generator enabled runner = CliRunner() result = runner.invoke( main, ["--url", "https://akamhy.github.io", "--known-urls", "--file"] ) assert result.exit_code == 0 assert result.output.count("\n") > 40 assert result.output.count("akamhy.github.io") > 40 assert result.output.find("in the current working directory.\n") != -1 # without file runner = CliRunner() result = runner.invoke(main, ["--url", "https://akamhy.github.io", "--known-urls"]) assert result.exit_code == 0 assert result.output.count("\n") > 40 assert result.output.count("akamhy.github.io") > 40 waybackpy-3.0.6/tests/test_save_api.py000066400000000000000000000206661421412532700200670ustar00rootroot00000000000000import random import string import time from datetime import datetime from typing import cast import pytest from requests.structures import CaseInsensitiveDict from waybackpy.exceptions import MaximumSaveRetriesExceeded from waybackpy.save_api import WaybackMachineSaveAPI def rndstr(n: int) -> str: return "".join( random.choice(string.ascii_uppercase + string.digits) for _ in range(n) ) def test_save() -> None: url = "https://github.com/akamhy/waybackpy" user_agent = ( "Mozilla/5.0 (MacBook Air; M1 Mac OS X 11_4) AppleWebKit/605.1.15 " "(KHTML, like Gecko) Version/14.1.1 Safari/604.1" ) save_api = WaybackMachineSaveAPI(url, user_agent) save_api.save() archive_url = save_api.archive_url timestamp = save_api.timestamp() headers = save_api.headers # CaseInsensitiveDict cached_save = save_api.cached_save assert cached_save in [True, False] assert archive_url.find("github.com/akamhy/waybackpy") != -1 assert timestamp is not None assert str(headers).find("github.com/akamhy/waybackpy") != -1 assert isinstance(save_api.timestamp(), datetime) def test_max_redirect_exceeded() -> None: with pytest.raises(MaximumSaveRetriesExceeded): url = f"https://{rndstr}.gov" user_agent = ( "Mozilla/5.0 (MacBook Air; M1 Mac OS X 11_4) AppleWebKit/605.1.15 " "(KHTML, like Gecko) Version/14.1.1 Safari/604.1" ) save_api = WaybackMachineSaveAPI(url, user_agent, max_tries=3) save_api.save() def test_sleep() -> None: """ sleeping is actually very important for SaveAPI interface stability. The test checks that the time taken by sleep method is as intended. """ url = "https://example.com" user_agent = ( "Mozilla/5.0 (MacBook Air; M1 Mac OS X 11_4) AppleWebKit/605.1.15 " "(KHTML, like Gecko) Version/14.1.1 Safari/604.1" ) save_api = WaybackMachineSaveAPI(url, user_agent) s_time = int(time.time()) save_api.sleep(6) # multiple of 3 sleep for 10 seconds e_time = int(time.time()) assert (e_time - s_time) >= 10 s_time = int(time.time()) save_api.sleep(7) # sleeps for 5 seconds e_time = int(time.time()) assert (e_time - s_time) >= 5 def test_timestamp() -> None: url = "https://example.com" user_agent = ( "Mozilla/5.0 (MacBook Air; M1 Mac OS X 11_4) AppleWebKit/605.1.15 " "(KHTML, like Gecko) Version/14.1.1 Safari/604.1" ) save_api = WaybackMachineSaveAPI(url, user_agent) now = datetime.utcnow().strftime("%Y%m%d%H%M%S") save_api._archive_url = f"https://web.archive.org/web/{now}/{url}/" save_api.timestamp() assert save_api.cached_save is False now = "20100124063622" save_api._archive_url = f"https://web.archive.org/web/{now}/{url}/" save_api.timestamp() assert save_api.cached_save is True def test_archive_url_parser() -> None: """ Testing three regex for matches and also tests the response URL. """ url = "https://example.com" user_agent = ( "Mozilla/5.0 (MacBook Air; M1 Mac OS X 11_4) AppleWebKit/605.1.15 " "(KHTML, like Gecko) Version/14.1.1 Safari/604.1" ) save_api = WaybackMachineSaveAPI(url, user_agent) h = ( "\nSTART\nContent-Location: " "/web/20201126185327/https://www.scribbr.com/citing-sources/et-al" "\nEND\n" ) save_api.headers = h # type: ignore[assignment] expected_url = ( "https://web.archive.org/web/20201126185327/" "https://www.scribbr.com/citing-sources/et-al" ) assert save_api.archive_url_parser() == expected_url headers = { "Server": "nginx/1.15.8", "Date": "Sat, 02 Jan 2021 09:40:25 GMT", "Content-Type": "text/html; charset=UTF-8", "Transfer-Encoding": "chunked", "Connection": "keep-alive", "X-Archive-Orig-Server": "nginx", "X-Archive-Orig-Date": "Sat, 02 Jan 2021 09:40:09 GMT", "X-Archive-Orig-Transfer-Encoding": "chunked", "X-Archive-Orig-Connection": "keep-alive", "X-Archive-Orig-Vary": "Accept-Encoding", "X-Archive-Orig-Last-Modified": "Fri, 01 Jan 2021 12:19:00 GMT", "X-Archive-Orig-Strict-Transport-Security": "max-age=31536000, max-age=0;", "X-Archive-Guessed-Content-Type": "text/html", "X-Archive-Guessed-Charset": "utf-8", "Memento-Datetime": "Sat, 02 Jan 2021 09:40:09 GMT", "Link": ( '; rel="original", ' "; rel="timemap"; type="application/link-format", ' "; rel="timegate", ; rel="first memento"; ' 'datetime="Mon, 01 Jun 2020 08:29:11 GMT", ; " 'rel="prev memento"; datetime="Thu, 26 Nov 2020 18:53:27 GMT", ' "; rel="memento"; datetime="Sat, 02 Jan 2021 ' '09:40:09 GMT", ; " 'rel="last memento"; datetime="Sat, 02 Jan 2021 09:40:09 GMT"' ), "Content-Security-Policy": ( "default-src 'self' 'unsafe-eval' 'unsafe-inline' " "data: blob: archive.org web.archive.org analytics.archive.org " "pragma.archivelab.org", ), "X-Archive-Src": "spn2-20210102092956-wwwb-spn20.us.archive.org-8001.warc.gz", "Server-Timing": ( "captures_list;dur=112.646325, exclusion.robots;dur=0.172010, " "exclusion.robots.policy;dur=0.158205, RedisCDXSource;dur=2.205932, " "esindex;dur=0.014647, LoadShardBlock;dur=82.205012, " "PetaboxLoader3.datanode;dur=70.750239, CDXLines.iter;dur=24.306278, " "load_resource;dur=26.520179" ), "X-App-Server": "wwwb-app200", "X-ts": "200", "X-location": "All", "X-Cache-Key": ( "httpsweb.archive.org/web/20210102094009/" "https://www.scribbr.com/citing-sources/et-al/IN", ), "X-RL": "0", "X-Page-Cache": "MISS", "X-Archive-Screenname": "0", "Content-Encoding": "gzip", } save_api.headers = cast(CaseInsensitiveDict[str], headers) expected_url2 = ( "https://web.archive.org/web/20210102094009/" "https://www.scribbr.com/citing-sources/et-al/" ) assert save_api.archive_url_parser() == expected_url2 expected_url_3 = ( "https://web.archive.org/web/20171128185327/" "https://www.scribbr.com/citing-sources/et-al/US" ) h = f"START\nX-Cache-Key: {expected_url_3}\nEND\n" save_api.headers = h # type: ignore[assignment] expected_url4 = ( "https://web.archive.org/web/20171128185327/" "https://www.scribbr.com/citing-sources/et-al/" ) assert save_api.archive_url_parser() == expected_url4 h = "TEST TEST TEST AND NO MATCH - TEST FOR RESPONSE URL MATCHING" save_api.headers = h # type: ignore[assignment] save_api.response_url = ( "https://web.archive.org/web/20171128185327/" "https://www.scribbr.com/citing-sources/et-al" ) expected_url5 = ( "https://web.archive.org/web/20171128185327/" "https://www.scribbr.com/citing-sources/et-al" ) assert save_api.archive_url_parser() == expected_url5 def test_archive_url() -> None: """ Checks the attribute archive_url's value when the save method was not explicitly invoked by the end-user but the save method was invoked implicitly by the archive_url method which is an attribute due to @property. """ url = "https://example.com" user_agent = ( "Mozilla/5.0 (MacBook Air; M1 Mac OS X 11_4) AppleWebKit/605.1.15 " "(KHTML, like Gecko) Version/14.1.1 Safari/604.1" ) save_api = WaybackMachineSaveAPI(url, user_agent) save_api.saved_archive = ( "https://web.archive.org/web/20220124063056/https://example.com/" ) save_api._archive_url = save_api.saved_archive assert save_api.archive_url == save_api.saved_archive waybackpy-3.0.6/tests/test_utils.py000066400000000000000000000003641421412532700174310ustar00rootroot00000000000000from waybackpy import __version__ from waybackpy.utils import DEFAULT_USER_AGENT def test_default_user_agent() -> None: assert ( DEFAULT_USER_AGENT == f"waybackpy {__version__} - https://github.com/akamhy/waybackpy" ) waybackpy-3.0.6/tests/test_wrapper.py000066400000000000000000000025751421412532700177570ustar00rootroot00000000000000from waybackpy.wrapper import Url def test_oldest() -> None: url = "https://bing.com" oldest_archive = ( "https://web.archive.org/web/20030726111100/http://www.bing.com:80/" ) wayback = Url(url).oldest() assert wayback.archive_url == oldest_archive assert str(wayback) == oldest_archive assert len(wayback) > 365 * 15 # days in a year times years def test_newest() -> None: url = "https://www.youtube.com/" wayback = Url(url).newest() assert "youtube" in str(wayback.archive_url) assert "archived_snapshots" in str(wayback.json) def test_near() -> None: url = "https://www.google.com" wayback = Url(url).near(year=2010, month=10, day=10, hour=10, minute=10) assert "20101010" in str(wayback.archive_url) def test_total_archives() -> None: wayback = Url("https://akamhy.github.io") assert wayback.total_archives() > 10 wayback = Url("https://gaha.ef4i3n.m5iai3kifp6ied.cima/gahh2718gs/ahkst63t7gad8") assert wayback.total_archives() == 0 def test_known_urls() -> None: wayback = Url("akamhy.github.io") assert len(list(wayback.known_urls(subdomain=True))) > 40 def test_Save() -> None: wayback = Url("https://en.wikipedia.org/wiki/Asymptotic_equipartition_property") wayback.save() archive_url = str(wayback.archive_url) assert archive_url.find("Asymptotic_equipartition_property") != -1 waybackpy-3.0.6/waybackpy/000077500000000000000000000000001421412532700155055ustar00rootroot00000000000000waybackpy-3.0.6/waybackpy/__init__.py000066400000000000000000000006271421412532700176230ustar00rootroot00000000000000"""Module initializer and provider of static information.""" __version__ = "3.0.6" from .availability_api import WaybackMachineAvailabilityAPI from .cdx_api import WaybackMachineCDXServerAPI from .save_api import WaybackMachineSaveAPI from .wrapper import Url __all__ = [ "__version__", "WaybackMachineAvailabilityAPI", "WaybackMachineCDXServerAPI", "WaybackMachineSaveAPI", "Url", ] waybackpy-3.0.6/waybackpy/availability_api.py000066400000000000000000000220221421412532700213600ustar00rootroot00000000000000""" This module interfaces the Wayback Machine's availability API. The interface is useful for looking up archives and finding archives that are close to a specific date and time. It has a class WaybackMachineAvailabilityAPI, and the class has methods like: near() for retrieving archives close to a specific date and time. oldest() for retrieving the first archive URL of the webpage. newest() for retrieving the latest archive of the webpage. The Wayback Machine Availability API response must be a valid JSON and if it is not then an exception, InvalidJSONInAvailabilityAPIResponse is raised. If the Availability API returned valid JSON but archive URL could not be found it it then ArchiveNotInAvailabilityAPIResponse is raised. """ import json import time from datetime import datetime from typing import Any, Dict, Optional import requests from requests.models import Response from .exceptions import ( ArchiveNotInAvailabilityAPIResponse, InvalidJSONInAvailabilityAPIResponse, ) from .utils import ( DEFAULT_USER_AGENT, unix_timestamp_to_wayback_timestamp, wayback_timestamp, ) ResponseJSON = Dict[str, Any] class WaybackMachineAvailabilityAPI: """ Class that interfaces the Wayback Machine's availability API. """ def __init__( self, url: str, user_agent: str = DEFAULT_USER_AGENT, max_tries: int = 3 ) -> None: self.url = str(url).strip().replace(" ", "%20") self.user_agent = user_agent self.headers: Dict[str, str] = {"User-Agent": self.user_agent} self.payload: Dict[str, str] = {"url": self.url} self.endpoint: str = "https://archive.org/wayback/available" self.max_tries: int = max_tries self.tries: int = 0 self.last_api_call_unix_time: int = int(time.time()) self.api_call_time_gap: int = 5 self.json: Optional[ResponseJSON] = None self.response: Optional[Response] = None def __repr__(self) -> str: """ Same as string representation, just return the archive URL as a string. """ return str(self) def __str__(self) -> str: """ String representation of the class. If atleast one API call was successfully made then return the archive URL as a string. Else returns "" (empty string literal). """ # __str__ can not return anything other than a string object # So, if a string repr is asked even before making a API request # just return "" if not self.json: return "" return self.archive_url def setup_json(self) -> Optional[ResponseJSON]: """ Makes the API call to the availability API and set the JSON response to the JSON attribute of the instance and also returns the JSON attribute. time_diff and sleep_time makes sure that you are not making too many requests in a short interval of item, making too many requests is bad as Wayback Machine may reject them above a certain threshold. The end-user can change the api_call_time_gap attribute of the instance to increase or decrease the default time gap between two successive API calls, but it is not recommended to increase it. """ time_diff = int(time.time()) - self.last_api_call_unix_time sleep_time = self.api_call_time_gap - time_diff if sleep_time > 0: time.sleep(sleep_time) self.response = requests.get( self.endpoint, params=self.payload, headers=self.headers ) self.last_api_call_unix_time = int(time.time()) self.tries += 1 try: self.json = None if self.response is None else self.response.json() except json.decoder.JSONDecodeError as json_decode_error: raise InvalidJSONInAvailabilityAPIResponse( f"Response data:\n{self.response.text}" ) from json_decode_error return self.json def timestamp(self) -> datetime: """ Converts the timestamp form the JSON response to datetime object. If JSON attribute of the instance is None it implies that the either the the last API call failed or one was never made. If not JSON or if JSON but no timestamp in the JSON response then returns the maximum value for datetime object that is possible. If you get an URL as a response form the availability API it is guaranteed that you can get the datetime object from the timestamp. """ if self.json is None or "archived_snapshots" not in self.json: return datetime.max if ( self.json is not None and "archived_snapshots" in self.json and self.json["archived_snapshots"] is not None and "closest" in self.json["archived_snapshots"] and self.json["archived_snapshots"]["closest"] is not None and "timestamp" in self.json["archived_snapshots"]["closest"] ): return datetime.strptime( self.json["archived_snapshots"]["closest"]["timestamp"], "%Y%m%d%H%M%S" ) raise ValueError("Timestamp not found in the Availability API's JSON response.") @property def archive_url(self) -> str: """ Reads the the JSON response data and returns the timestamp if found and if not found raises ArchiveNotInAvailabilityAPIResponse. """ archive_url = "" data = self.json # If the user didn't invoke oldest, newest or near but tries to access # archive_url attribute then assume they that are fine with any archive # and invoke the oldest method. if not data: self.oldest() # If data is still not none then probably there are no # archive for the requested URL. if not data or not data["archived_snapshots"]: while (self.tries < self.max_tries) and ( not data or not data["archived_snapshots"] ): self.setup_json() # It makes a new API call data = self.json # setup_json() updates value of json attribute # If exhausted max_tries, then give up and # raise ArchiveNotInAvailabilityAPIResponse. if not data or not data["archived_snapshots"]: raise ArchiveNotInAvailabilityAPIResponse( "Archive not found in the availability " "API response, the URL you requested may not have any archives " "yet. You may retry after some time or archive the webpage now.\n" "Response data:\n" "" if self.response is None else self.response.text ) else: archive_url = data["archived_snapshots"]["closest"]["url"] archive_url = archive_url.replace( "http://web.archive.org/web/", "https://web.archive.org/web/", 1 ) return archive_url def oldest(self) -> "WaybackMachineAvailabilityAPI": """ Passes the date 1994-01-01 to near which should return the oldest archive because Wayback Machine was started in May, 1996 and it is assumed that there would be no archive older than January 1, 1994. """ return self.near(year=1994, month=1, day=1) def newest(self) -> "WaybackMachineAvailabilityAPI": """ Passes the current UNIX time to near() for retrieving the newest archive from the availability API. Remember UNIX time is UTC and Wayback Machine is also UTC based. """ return self.near(unix_timestamp=int(time.time())) def near( self, year: Optional[int] = None, month: Optional[int] = None, day: Optional[int] = None, hour: Optional[int] = None, minute: Optional[int] = None, unix_timestamp: Optional[int] = None, ) -> "WaybackMachineAvailabilityAPI": """ The most important method of this Class, oldest() and newest() are dependent on it. It generates the timestamp based on the input either by calling the unix_timestamp_to_wayback_timestamp or wayback_timestamp method with appropriate arguments for their respective parameters. Adds the timestamp to the payload dictionary. And finally invokes the setup_json method to make the API call then finally returns the instance. """ if unix_timestamp: timestamp = unix_timestamp_to_wayback_timestamp(unix_timestamp) else: now = datetime.utcnow().timetuple() timestamp = wayback_timestamp( year=now.tm_year if year is None else year, month=now.tm_mon if month is None else month, day=now.tm_mday if day is None else day, hour=now.tm_hour if hour is None else hour, minute=now.tm_min if minute is None else minute, ) self.payload["timestamp"] = timestamp self.setup_json() return self waybackpy-3.0.6/waybackpy/cdx_api.py000066400000000000000000000273641421412532700175020ustar00rootroot00000000000000""" This module interfaces the Wayback Machine's CDX server API. The module has WaybackMachineCDXServerAPI which should be used by the users of this module to consume the CDX server API. WaybackMachineCDXServerAPI has a snapshot method that yields the snapshots, and the snapshots are yielded as instances of the CDXSnapshot class. """ import time from datetime import datetime from typing import Dict, Generator, List, Optional, Union, cast from .cdx_snapshot import CDXSnapshot from .cdx_utils import ( check_collapses, check_filters, check_match_type, check_sort, full_url, get_response, get_total_pages, ) from .exceptions import NoCDXRecordFound, WaybackError from .utils import ( DEFAULT_USER_AGENT, unix_timestamp_to_wayback_timestamp, wayback_timestamp, ) class WaybackMachineCDXServerAPI: """ Class that interfaces the CDX server API of the Wayback Machine. snapshot() returns a generator that can be iterated upon by the end-user, the generator returns the snapshots/entries as instance of CDXSnapshot to make the usage easy, just use '.' to get any attribute as the attributes are accessible via a dot ".". """ # start_timestamp: from, can not use from as it's a keyword # end_timestamp: to, not using to as can not use from def __init__( self, url: str, user_agent: str = DEFAULT_USER_AGENT, start_timestamp: Optional[str] = None, end_timestamp: Optional[str] = None, filters: Optional[List[str]] = None, match_type: Optional[str] = None, sort: Optional[str] = None, gzip: Optional[str] = None, collapses: Optional[List[str]] = None, limit: Optional[str] = None, max_tries: int = 3, use_pagination: bool = False, closest: Optional[str] = None, ) -> None: self.url = str(url).strip().replace(" ", "%20") self.user_agent = user_agent self.start_timestamp = None if start_timestamp is None else str(start_timestamp) self.end_timestamp = None if end_timestamp is None else str(end_timestamp) self.filters = [] if filters is None else filters check_filters(self.filters) self.match_type = None if match_type is None else str(match_type).strip() check_match_type(self.match_type, self.url) self.sort = None if sort is None else str(sort).strip() check_sort(self.sort) self.gzip = gzip self.collapses = [] if collapses is None else collapses check_collapses(self.collapses) self.limit = 25000 if limit is None else limit self.max_tries = max_tries self.use_pagination = use_pagination self.closest = None if closest is None else str(closest) self.last_api_request_url: Optional[str] = None self.endpoint = "https://web.archive.org/cdx/search/cdx" def cdx_api_manager( self, payload: Dict[str, str], headers: Dict[str, str] ) -> Generator[str, None, None]: """ This method uses the pagination API of the CDX server if use_pagination attribute is True else uses the standard CDX server response data. """ # When using the pagination API of the CDX server. if self.use_pagination is True: total_pages = get_total_pages(self.url, self.user_agent) successive_blank_pages = 0 for i in range(total_pages): payload["page"] = str(i) url = full_url(self.endpoint, params=payload) res = get_response(url, headers=headers) if isinstance(res, Exception): raise res self.last_api_request_url = url text = res.text # Reset the counter if the last page was blank # but the current page is not. if successive_blank_pages == 1: if len(text) != 0: successive_blank_pages = 0 # Increase the succesive page counter on encountering # blank page. if len(text) == 0: successive_blank_pages += 1 # If two succesive pages are blank # then we don't have any more pages left to # iterate. if successive_blank_pages >= 2: break yield text # When not using the pagination API of the CDX server else: payload["showResumeKey"] = "true" payload["limit"] = str(self.limit) resume_key = None more = True while more: if resume_key: payload["resumeKey"] = resume_key url = full_url(self.endpoint, params=payload) res = get_response(url, headers=headers) if isinstance(res, Exception): raise res self.last_api_request_url = url text = res.text.strip() lines = text.splitlines() more = False if len(lines) >= 3: second_last_line = lines[-2] if len(second_last_line) == 0: resume_key = lines[-1].strip() text = text.replace(resume_key, "", 1).strip() more = True yield text def add_payload(self, payload: Dict[str, str]) -> None: """ Adds the payload to the payload dictionary. """ if self.start_timestamp: payload["from"] = self.start_timestamp if self.end_timestamp: payload["to"] = self.end_timestamp if self.gzip is None: payload["gzip"] = "false" if self.closest: payload["closest"] = self.closest if self.match_type: payload["matchType"] = self.match_type if self.sort: payload["sort"] = self.sort if self.filters and len(self.filters) > 0: for i, _filter in enumerate(self.filters): payload["filter" + str(i)] = _filter if self.collapses and len(self.collapses) > 0: for i, collapse in enumerate(self.collapses): payload["collapse" + str(i)] = collapse payload["url"] = self.url def near( self, year: Optional[int] = None, month: Optional[int] = None, day: Optional[int] = None, hour: Optional[int] = None, minute: Optional[int] = None, unix_timestamp: Optional[int] = None, wayback_machine_timestamp: Optional[Union[int, str]] = None, ) -> CDXSnapshot: """ Fetch archive close to a datetime, it can only return a single URL. If you want more do not use this method instead use the class. """ if unix_timestamp: timestamp = unix_timestamp_to_wayback_timestamp(unix_timestamp) elif wayback_machine_timestamp: timestamp = str(wayback_machine_timestamp) else: now = datetime.utcnow().timetuple() timestamp = wayback_timestamp( year=now.tm_year if year is None else year, month=now.tm_mon if month is None else month, day=now.tm_mday if day is None else day, hour=now.tm_hour if hour is None else hour, minute=now.tm_min if minute is None else minute, ) self.closest = timestamp self.sort = "closest" self.limit = 1 first_snapshot = None for snapshot in self.snapshots(): first_snapshot = snapshot break if not first_snapshot: raise NoCDXRecordFound( "Wayback Machine's CDX server did not return any records " + "for the query. The URL may not have any archives " + " on the Wayback Machine or the URL may have been recently " + "archived and is still not available on the CDX server." ) return first_snapshot def newest(self) -> CDXSnapshot: """ Passes the current UNIX time to near() for retrieving the newest archive from the availability API. Remember UNIX time is UTC and Wayback Machine is also UTC based. """ return self.near(unix_timestamp=int(time.time())) def oldest(self) -> CDXSnapshot: """ Passes the date 1994-01-01 to near which should return the oldest archive because Wayback Machine was started in May, 1996 and it is assumed that there would be no archive older than January 1, 1994. """ return self.near(year=1994, month=1, day=1) def snapshots(self) -> Generator[CDXSnapshot, None, None]: """ This function yields the CDX data lines as snapshots. As it is a generator it exhaustible, the reason that this is a generator and not a list are: a) CDX server API can return millions of entries for a query and list is not suitable for such cases. b) Preventing memory usage issues, as told before this method may yield millions of records for some queries and your system may not have enough memory for such a big list. Also Remember this if outputing to Jupyter Notebooks. The objects yielded by this method are instance of CDXSnapshot class, you can access the attributes of the entries as the attribute of the instance itself. """ payload: Dict[str, str] = {} headers = {"User-Agent": self.user_agent} self.add_payload(payload) entries = self.cdx_api_manager(payload, headers) for entry in entries: if entry.isspace() or len(entry) <= 1 or not entry: continue # each line is a snapshot aka entry of the CDX server API. # We are able to split the page by lines because it only # splits the lines on a sinlge page and not all the entries # at once, thus there should be no issues of too much memory usage. snapshot_list = entry.split("\n") for snapshot in snapshot_list: # 14 + 32 == 46 ( timestamp + digest ), ignore the invalid entries. # they are invalid if their length is smaller than sum of length # of a standard wayback_timestamp and standard digest of an entry. if len(snapshot) < 46: continue properties: Dict[str, Optional[str]] = { "urlkey": None, "timestamp": None, "original": None, "mimetype": None, "statuscode": None, "digest": None, "length": None, } property_value = snapshot.split(" ") total_property_values = len(property_value) warranted_total_property_values = len(properties) if total_property_values != warranted_total_property_values: raise WaybackError( f"Snapshot returned by CDX API has {total_property_values} prop" f"erties instead of expected {warranted_total_property_values} " f"properties.\nProblematic Snapshot: {snapshot}" ) ( properties["urlkey"], properties["timestamp"], properties["original"], properties["mimetype"], properties["statuscode"], properties["digest"], properties["length"], ) = property_value yield CDXSnapshot(cast(Dict[str, str], properties)) waybackpy-3.0.6/waybackpy/cdx_snapshot.py000066400000000000000000000063641421412532700205650ustar00rootroot00000000000000""" Module that contains the CDXSnapshot class, CDX records/lines are casted to CDXSnapshot objects for easier access. The CDX index format is plain text data. Each line ('record') indicates a crawled document. And these lines are casted to CDXSnapshot. """ from datetime import datetime from typing import Dict class CDXSnapshot: """ Class for the CDX snapshot lines('record') returned by the CDX API, Each valid line of the CDX API is casted to an CDXSnapshot object by the CDX API interface, just use "." to access any attribute of the CDX server API snapshot. This provides the end-user the ease of using the data as attributes of the CDXSnapshot. The string representation of the class is identical to the line returned by the CDX server API. Besides all the attributes of the CDX server API this class also provides archive_url attribute, yes it is the archive url of the snapshot. Attributes of the this class and what they represents and are useful for: urlkey: The document captured, expressed as a SURT SURT stands for Sort-friendly URI Reordering Transform, and is a transformation applied to URIs which makes their left-to-right representation better match the natural hierarchy of domain names. A URI has SURT form . timestamp: The timestamp of the archive, format is yyyyMMddhhmmss and type is string. datetime_timestamp: The timestamp as a datetime object. original: The original URL of the archive. If archive_url is https://web.archive.org/web/20220113130051/https://google.com then the original URL is https://google.com mimetype: The document’s file type. e.g. text/html statuscode: HTTP response code for the document at the time of its crawling digest: Base32-encoded SHA-1 checksum of the document for discriminating with others length: Document’s volume of bytes in the WARC file archive_url: The archive url of the snapshot, this is not returned by the CDX server API but created by this class on init. """ def __init__(self, properties: Dict[str, str]) -> None: self.urlkey: str = properties["urlkey"] self.timestamp: str = properties["timestamp"] self.datetime_timestamp: datetime = datetime.strptime( self.timestamp, "%Y%m%d%H%M%S" ) self.original: str = properties["original"] self.mimetype: str = properties["mimetype"] self.statuscode: str = properties["statuscode"] self.digest: str = properties["digest"] self.length: str = properties["length"] self.archive_url: str = ( f"https://web.archive.org/web/{self.timestamp}/{self.original}" ) def __repr__(self) -> str: """ Same as __str__() """ return str(self) def __str__(self) -> str: """ The string representation is same as the line returned by the CDX server API for the snapshot. """ return ( f"{self.urlkey} {self.timestamp} {self.original} " f"{self.mimetype} {self.statuscode} {self.digest} {self.length}" ) waybackpy-3.0.6/waybackpy/cdx_utils.py000066400000000000000000000137311421412532700200620ustar00rootroot00000000000000""" Utility functions required for accessing the CDX server API. These are here in this module so that we don’t make any module too long. """ import re from typing import Any, Dict, List, Optional, Union from urllib.parse import quote import requests from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry from .exceptions import BlockedSiteError, WaybackError from .utils import DEFAULT_USER_AGENT def get_total_pages(url: str, user_agent: str = DEFAULT_USER_AGENT) -> int: """ When using the pagination use adding showNumPages=true to the request URL makes the CDX server return an integer which is the number of pages of CDX pages available for us to query using the pagination API. """ endpoint = "https://web.archive.org/cdx/search/cdx?" payload = {"showNumPages": "true", "url": str(url)} headers = {"User-Agent": user_agent} request_url = full_url(endpoint, params=payload) response = get_response(request_url, headers=headers) check_for_blocked_site(response, url) if isinstance(response, requests.Response): return int(response.text.strip()) raise response def check_for_blocked_site( response: Union[requests.Response, Exception], url: Optional[str] = None ) -> None: """ Checks that the URL can be archived by wayback machine or not. robots.txt policy of the site may prevent the wayback machine. """ # see https://github.com/akamhy/waybackpy/issues/157 # the following if block is to make mypy happy. if isinstance(response, Exception): raise response if not url: url = "The requested content" if ( "org.archive.util.io.RuntimeIOException: " + "org.archive.wayback.exception.AdministrativeAccessControlException: " + "Blocked Site Error" in response.text.strip() ): raise BlockedSiteError( f"{url} is excluded from Wayback Machine by the site's robots.txt policy." ) def full_url(endpoint: str, params: Dict[str, Any]) -> str: """ As the function's name already implies that it returns full URL, but why we need a function for generating full URL? The CDX server can support multiple arguments for parameters such as filter and collapse and this function adds them without overwriting earlier added arguments. """ if not params: return endpoint _full_url = endpoint if endpoint.endswith("?") else (endpoint + "?") for key, val in params.items(): key = "filter" if key.startswith("filter") else key key = "collapse" if key.startswith("collapse") else key amp = "" if _full_url.endswith("?") else "&" val = quote(str(val), safe="") _full_url += f"{amp}{key}={val}" return _full_url def get_response( url: str, headers: Optional[Dict[str, str]] = None, retries: int = 5, backoff_factor: float = 0.5, ) -> Union[requests.Response, Exception]: """ Makes get request to the CDX server and returns the response. """ session = requests.Session() retries_ = Retry( total=retries, backoff_factor=backoff_factor, status_forcelist=[500, 502, 503, 504], ) session.mount("https://", HTTPAdapter(max_retries=retries_)) response = session.get(url, headers=headers) session.close() check_for_blocked_site(response) return response def check_filters(filters: List[str]) -> None: """ Check that the filter arguments passed by the end-user are valid. If not valid then raise WaybackError. """ if not isinstance(filters, list): raise WaybackError("filters must be a list.") # [!]field:regex for _filter in filters: match = re.search( r"(\!?(?:urlkey|timestamp|original|mimetype|statuscode|digest|length)):" r"(.*)", _filter, ) if match is None or len(match.groups()) != 2: exc_message = f"Filter '{_filter}' is not following the cdx filter syntax." raise WaybackError(exc_message) def check_collapses(collapses: List[str]) -> bool: """ Check that the collapse arguments passed by the end-user are valid. If not valid then raise WaybackError. """ if not isinstance(collapses, list): raise WaybackError("collapses must be a list.") if len(collapses) == 0: return True for collapse in collapses: match = re.search( r"(urlkey|timestamp|original|mimetype|statuscode|digest|length)" r"(:?[0-9]{1,99})?", collapse, ) if match is None or len(match.groups()) != 2: exc_message = ( f"collapse argument '{collapse}' " "is not following the cdx collapse syntax." ) raise WaybackError(exc_message) return True def check_match_type(match_type: Optional[str], url: str) -> bool: """ Check that the match_type argument passed by the end-user is valid. If not valid then raise WaybackError. """ legal_match_type = ["exact", "prefix", "host", "domain"] if not match_type: return True if "*" in url: raise WaybackError( "Can not use wildcard in the URL along with the match_type arguments." ) if match_type not in legal_match_type: exc_message = ( f"{match_type} is not an allowed match type.\n" "Use one from 'exact', 'prefix', 'host' or 'domain'" ) raise WaybackError(exc_message) return True def check_sort(sort: Optional[str]) -> bool: """ Check that the sort argument passed by the end-user is valid. If not valid then raise WaybackError. """ legal_sort = ["default", "closest", "reverse"] if not sort: return True if sort not in legal_sort: exc_message = ( f"{sort} is not an allowed argument for sort.\n" "Use one from 'default', 'closest' or 'reverse'" ) raise WaybackError(exc_message) return True waybackpy-3.0.6/waybackpy/cli.py000066400000000000000000000321721421412532700166330ustar00rootroot00000000000000""" Module responsible for enabling waybackpy to function as a CLI tool. """ import os import random import re import string from typing import Any, Dict, Generator, List, Optional import click import requests from . import __version__ from .cdx_api import WaybackMachineCDXServerAPI from .exceptions import BlockedSiteError, NoCDXRecordFound from .save_api import WaybackMachineSaveAPI from .utils import DEFAULT_USER_AGENT from .wrapper import Url def handle_cdx_closest_derivative_methods( cdx_api: "WaybackMachineCDXServerAPI", oldest: bool, near: bool, newest: bool, near_args: Optional[Dict[str, int]] = None, ) -> None: """ Handles the closest parameter derivative methods. near, newest and oldest use the closest parameter with active closest based sorting. """ try: if near: if near_args: archive_url = cdx_api.near(**near_args).archive_url else: archive_url = cdx_api.near().archive_url elif newest: archive_url = cdx_api.newest().archive_url elif oldest: archive_url = cdx_api.oldest().archive_url click.echo("Archive URL:") click.echo(archive_url) except NoCDXRecordFound as exc: click.echo(click.style("NoCDXRecordFound: ", fg="red") + str(exc), err=True) except BlockedSiteError as exc: click.echo(click.style("BlockedSiteError: ", fg="red") + str(exc), err=True) def handle_cdx(data: List[Any]) -> None: """ Handles the CDX CLI options and output format. """ url = data[0] user_agent = data[1] start_timestamp = data[2] end_timestamp = data[3] cdx_filter = data[4] collapse = data[5] cdx_print = data[6] limit = data[7] gzip = data[8] match_type = data[9] sort = data[10] use_pagination = data[11] closest = data[12] filters = list(cdx_filter) collapses = list(collapse) cdx_print = list(cdx_print) cdx_api = WaybackMachineCDXServerAPI( url, user_agent=user_agent, start_timestamp=start_timestamp, end_timestamp=end_timestamp, closest=closest, filters=filters, match_type=match_type, sort=sort, use_pagination=use_pagination, gzip=gzip, collapses=collapses, limit=limit, ) snapshots = cdx_api.snapshots() for snapshot in snapshots: if len(cdx_print) == 0: click.echo(snapshot) else: output_string = [] if any(val in cdx_print for val in ["urlkey", "url-key", "url_key"]): output_string.append(snapshot.urlkey) if any( val in cdx_print for val in ["timestamp", "time-stamp", "time_stamp"] ): output_string.append(snapshot.timestamp) if "original" in cdx_print: output_string.append(snapshot.original) if any(val in cdx_print for val in ["mimetype", "mime-type", "mime_type"]): output_string.append(snapshot.mimetype) if any( val in cdx_print for val in ["statuscode", "status-code", "status_code"] ): output_string.append(snapshot.statuscode) if "digest" in cdx_print: output_string.append(snapshot.digest) if "length" in cdx_print: output_string.append(snapshot.length) if any( val in cdx_print for val in ["archiveurl", "archive-url", "archive_url"] ): output_string.append(snapshot.archive_url) click.echo(" ".join(output_string)) def save_urls_on_file(url_gen: Generator[str, None, None]) -> None: """ Save output of CDX API on file. Mainly here because of backwards compatibility. """ domain = None sys_random = random.SystemRandom() uid = "".join( sys_random.choice(string.ascii_lowercase + string.digits) for _ in range(6) ) url_count = 0 file_name = None for url in url_gen: url_count += 1 if not domain: match = re.search("https?://([A-Za-z_0-9.-]+).*", url) domain = "domain-unknown" if match: domain = match.group(1) file_name = f"{domain}-urls-{uid}.txt" file_path = os.path.join(os.getcwd(), file_name) if not os.path.isfile(file_path): with open(file_path, "w+", encoding="utf-8") as file: file.close() with open(file_path, "a", encoding="utf-8") as file: file.write(f"{url}\n") click.echo(url) if url_count > 0: click.echo( f"\n\n{url_count} URLs saved inside '{file_name}' in the current " + "working directory." ) else: click.echo("No known URLs found. Please try a diffrent input!") @click.command() @click.option( "-u", "--url", help="URL on which Wayback machine operations are to be performed." ) @click.option( "-ua", "--user-agent", "--user_agent", default=DEFAULT_USER_AGENT, help=f"User agent, default value is '{DEFAULT_USER_AGENT}'.", ) @click.option("-v", "--version", is_flag=True, default=False, help="waybackpy version.") @click.option( "-l", "--show-license", "--show_license", "--license", is_flag=True, default=False, help="Show license of Waybackpy.", ) @click.option( "-n", "--newest", "-au", "--archive_url", "--archive-url", default=False, is_flag=True, help="Retrieve the newest archive of URL.", ) @click.option( "-o", "--oldest", default=False, is_flag=True, help="Retrieve the oldest archive of URL.", ) @click.option( "-N", "--near", default=False, is_flag=True, help="Archive close to a specified time.", ) @click.option("-Y", "--year", type=click.IntRange(1994, 9999), help="Year in integer.") @click.option("-M", "--month", type=click.IntRange(1, 12), help="Month in integer.") @click.option("-D", "--day", type=click.IntRange(1, 31), help="Day in integer.") @click.option("-H", "--hour", type=click.IntRange(0, 24), help="Hour in integer.") @click.option("-MIN", "--minute", type=click.IntRange(0, 60), help="Minute in integer.") @click.option( "-s", "--save", default=False, is_flag=True, help="Save the specified URL's webpage and print the archive URL.", ) @click.option( "-h", "--headers", default=False, is_flag=True, help="Headers data of the SavePageNow API.", ) @click.option( "-ku", "--known-urls", "--known_urls", default=False, is_flag=True, help="List known URLs. Uses CDX API.", ) @click.option( "-sub", "--subdomain", default=False, is_flag=True, help="Use with '--known_urls' to include known URLs for subdomains.", ) @click.option( "-f", "--file", default=False, is_flag=True, help="Use with '--known_urls' to save the URLs in file at current directory.", ) @click.option( "--cdx", default=False, is_flag=True, help="Flag for using CDX API.", ) @click.option( "-st", "--start-timestamp", "--start_timestamp", "--from", help="Start timestamp for CDX API in yyyyMMddhhmmss format.", ) @click.option( "-et", "--end-timestamp", "--end_timestamp", "--to", help="End timestamp for CDX API in yyyyMMddhhmmss format.", ) @click.option( "-C", "--closest", help="Archive that are closest the timestamp passed as arguments to this " + "parameter.", ) @click.option( "-f", "--cdx-filter", "--cdx_filter", "--filter", multiple=True, help="Filter on a specific field or all the CDX fields.", ) @click.option( "-mt", "--match-type", "--match_type", help="The default behavior is to return matches for an exact URL. " + "However, the CDX server can also return results matching a certain prefix, " + "a certain host, or all sub-hosts by using the match_type", ) @click.option( "-st", "--sort", help="Choose one from default, closest or reverse. It returns sorted CDX entries " + "in the response.", ) @click.option( "-up", "--use-pagination", "--use_pagination", default=False, is_flag=True, help="Use the pagination API of the CDX server instead of the default one.", ) @click.option( "-gz", "--gzip", help="To disable gzip compression pass false as argument to this parameter. " + "The default behavior is gzip compression enabled.", ) @click.option( "-c", "--collapse", multiple=True, help="Filtering or 'collapse' results based on a field, or a substring of a field.", ) @click.option( "-l", "--limit", help="Number of maximum record that CDX API is asked to return per API call, " + "default value is 25000 records.", ) @click.option( "-cp", "--cdx-print", "--cdx_print", multiple=True, help="Print only certain fields of the CDX API response, " + "if this parameter is not used then the plain text response of the CDX API " + "will be printed.", ) def main( # pylint: disable=no-value-for-parameter user_agent: str, version: bool, show_license: bool, newest: bool, oldest: bool, near: bool, save: bool, headers: bool, known_urls: bool, subdomain: bool, file: bool, cdx: bool, use_pagination: bool, cdx_filter: List[str], collapse: List[str], cdx_print: List[str], url: Optional[str] = None, year: Optional[int] = None, month: Optional[int] = None, day: Optional[int] = None, hour: Optional[int] = None, minute: Optional[int] = None, start_timestamp: Optional[str] = None, end_timestamp: Optional[str] = None, closest: Optional[str] = None, match_type: Optional[str] = None, sort: Optional[str] = None, gzip: Optional[str] = None, limit: Optional[str] = None, ) -> None: """\b _ _ | | | | __ ____ _ _ _| |__ __ _ ___| | ___ __ _ _ \\ \\ /\\ / / _` | | | | '_ \\ / _` |/ __| |/ / '_ \\| | | | \\ V V / (_| | |_| | |_) | (_| | (__| <| |_) | |_| | \\_/\\_/ \\__,_|\\__, |_.__/ \\__,_|\\___|_|\\_\\ .__/ \\__, | __/ | | | __/ | |___/ |_| |___/ Python package & CLI tool that interfaces the Wayback Machine APIs Repository: https://github.com/akamhy/waybackpy Documentation: https://github.com/akamhy/waybackpy/wiki/CLI-docs waybackpy - CLI usage(Demo video): https://asciinema.org/a/469890 Released under the MIT License. Use the flag --license for license. """ if version: click.echo(f"waybackpy version {__version__}") elif show_license: click.echo( requests.get( url="https://raw.githubusercontent.com/akamhy/waybackpy/master/LICENSE" ).text ) elif url is None: click.echo( click.style("NoURLDetected: ", fg="red") + "No URL detected. " + "Please provide an URL.", err=True, ) elif oldest: cdx_api = WaybackMachineCDXServerAPI(url, user_agent=user_agent) handle_cdx_closest_derivative_methods(cdx_api, oldest, near, newest) elif newest: cdx_api = WaybackMachineCDXServerAPI(url, user_agent=user_agent) handle_cdx_closest_derivative_methods(cdx_api, oldest, near, newest) elif near: cdx_api = WaybackMachineCDXServerAPI(url, user_agent=user_agent) near_args = {} keys = ["year", "month", "day", "hour", "minute"] args_arr = [year, month, day, hour, minute] for key, arg in zip(keys, args_arr): if arg: near_args[key] = arg handle_cdx_closest_derivative_methods( cdx_api, oldest, near, newest, near_args=near_args ) elif save: save_api = WaybackMachineSaveAPI(url, user_agent=user_agent) save_api.save() click.echo("Archive URL:") click.echo(save_api.archive_url) click.echo("Cached save:") click.echo(save_api.cached_save) if headers: click.echo("Save API headers:") click.echo(save_api.headers) elif known_urls: wayback = Url(url, user_agent) url_gen = wayback.known_urls(subdomain=subdomain) if file: save_urls_on_file(url_gen) else: for url_ in url_gen: click.echo(url_) elif cdx: data = [ url, user_agent, start_timestamp, end_timestamp, cdx_filter, collapse, cdx_print, limit, gzip, match_type, sort, use_pagination, closest, ] handle_cdx(data) else: click.echo( click.style("NoCommandFound: ", fg="red") + "Only URL passed, but did not specify what to do with the URL. " + "Use --help flag for help using waybackpy.", err=True, ) if __name__ == "__main__": main() # pylint: disable=no-value-for-parameter waybackpy-3.0.6/waybackpy/exceptions.py000066400000000000000000000026551421412532700202500ustar00rootroot00000000000000""" waybackpy.exceptions ~~~~~~~~~~~~~~~~~~~ This module contains the set of Waybackpy's exceptions. """ class WaybackError(Exception): """ Raised when Waybackpy can not return what you asked for. 1) Wayback Machine API Service is unreachable/down. 2) You passed illegal arguments. All other exceptions are inherited from this main exception. """ class NoCDXRecordFound(WaybackError): """ No records returned by the CDX server for a query. Raised when the user invokes near(), newest() or oldest() methods and there are no archives. """ class BlockedSiteError(WaybackError): """ Raised when the archives for website/URLs that was excluded from Wayback Machine are requested via the CDX server API. """ class TooManyRequestsError(WaybackError): """ Raised when you make more than 15 requests per minute and the Wayback Machine returns 429. See https://github.com/akamhy/waybackpy/issues/131 """ class MaximumRetriesExceeded(WaybackError): """ MaximumRetriesExceeded """ class MaximumSaveRetriesExceeded(MaximumRetriesExceeded): """ MaximumSaveRetriesExceeded """ class ArchiveNotInAvailabilityAPIResponse(WaybackError): """ Could not parse the archive in the JSON response of the availability API. """ class InvalidJSONInAvailabilityAPIResponse(WaybackError): """ availability api returned invalid JSON """ waybackpy-3.0.6/waybackpy/py.typed000066400000000000000000000000001421412532700171720ustar00rootroot00000000000000waybackpy-3.0.6/waybackpy/save_api.py000066400000000000000000000203221421412532700176450ustar00rootroot00000000000000""" This module interfaces the Wayback Machine's SavePageNow (SPN) API. The module has WaybackMachineSaveAPI class which should be used by the users of this module to use the SavePageNow API. """ import re import time from datetime import datetime from typing import Dict, Optional import requests from requests.adapters import HTTPAdapter from requests.models import Response from requests.structures import CaseInsensitiveDict from urllib3.util.retry import Retry from .exceptions import MaximumSaveRetriesExceeded, TooManyRequestsError, WaybackError from .utils import DEFAULT_USER_AGENT class WaybackMachineSaveAPI: """ WaybackMachineSaveAPI class provides an interface for saving URLs on the Wayback Machine. """ def __init__( self, url: str, user_agent: str = DEFAULT_USER_AGENT, max_tries: int = 8, ) -> None: self.url = str(url).strip().replace(" ", "%20") self.request_url = "https://web.archive.org/save/" + self.url self.user_agent = user_agent self.request_headers: Dict[str, str] = {"User-Agent": self.user_agent} if max_tries < 1: raise ValueError("max_tries should be positive") self.max_tries = max_tries self.total_save_retries = 5 self.backoff_factor = 0.5 self.status_forcelist = [500, 502, 503, 504] self._archive_url: Optional[str] = None self.instance_birth_time = datetime.utcnow() self.response: Optional[Response] = None self.headers: Optional[CaseInsensitiveDict[str]] = None self.status_code: Optional[int] = None self.response_url: Optional[str] = None self.cached_save: Optional[bool] = None self.saved_archive: Optional[str] = None @property def archive_url(self) -> str: """ Returns the archive URL is already cached by _archive_url else invoke the save method to save the archive which returns the archive thus we return the methods return value. """ if self._archive_url: return self._archive_url return self.save() def get_save_request_headers(self) -> None: """ Creates a session and tries 'retries' number of times to retrieve the archive. If successful in getting the response, sets the headers, status_code and response_url attributes. The archive is usually in the headers but it can also be the response URL as the Wayback Machine redirects to the archive after a successful capture of the webpage. Wayback Machine's save API is known to be very unreliable thus if it fails first check opening the response URL yourself in the browser. """ session = requests.Session() retries = Retry( total=self.total_save_retries, backoff_factor=self.backoff_factor, status_forcelist=self.status_forcelist, ) session.mount("https://", HTTPAdapter(max_retries=retries)) self.response = session.get(self.request_url, headers=self.request_headers) # requests.response.headers is requests.structures.CaseInsensitiveDict self.headers = self.response.headers self.status_code = self.response.status_code self.response_url = self.response.url session.close() if self.status_code == 429: # why wait 5 minutes and 429? # see https://github.com/akamhy/waybackpy/issues/97 raise TooManyRequestsError( f"Can not save '{self.url}'. " f"Save request refused by the server. " f"Save Page Now limits saving 15 URLs per minutes. " f"Try waiting for 5 minutes and then try again." ) # why 509? # see https://github.com/akamhy/waybackpy/pull/99 # also https://t.co/xww4YJ0Iwc if self.status_code == 509: raise WaybackError( f"Can not save '{self.url}'. You have probably reached the " f"limit of active sessions." ) def archive_url_parser(self) -> Optional[str]: """ Three regexen (like oxen?) are used to search for the archive URL in the headers and finally look in the response URL for the archive URL. """ regex1 = r"Content-Location: (/web/[0-9]{14}/.*)" match = re.search(regex1, str(self.headers)) if match: return "https://web.archive.org" + match.group(1) regex2 = r"rel=\"memento.*?(web\.archive\.org/web/[0-9]{14}/.*?)>" match = re.search(regex2, str(self.headers)) if match is not None and len(match.groups()) == 1: return "https://" + match.group(1) regex3 = r"X-Cache-Key:\shttps(.*)[A-Z]{2}" match = re.search(regex3, str(self.headers)) if match is not None and len(match.groups()) == 1: return "https" + match.group(1) self.response_url = ( "" if self.response_url is None else self.response_url.strip() ) regex4 = r"web\.archive\.org/web/(?:[0-9]*?)/(?:.*)$" match = re.search(regex4, self.response_url) if match is not None: return "https://" + match.group(0) return None @staticmethod def sleep(tries: int) -> None: """ Ensure that the we wait some time before succesive retries so that we don't waste the retries before the page is even captured by the Wayback Machine crawlers also ensures that we are not putting too much load on the Wayback Machine's save API. If tries are multiple of 3 sleep 10 seconds else sleep 5 seconds. """ sleep_seconds = 5 if tries % 3 == 0: sleep_seconds = 10 time.sleep(sleep_seconds) def timestamp(self) -> datetime: """ Read the timestamp off the archive URL and convert the Wayback Machine timestamp to datetime object. Also check if the time on archive is URL and compare it to instance birth time. If time on the archive is older than the instance creation time set the cached_save to True else set it to False. The flag can be used to check if the Wayback Machine didn't serve a Cached URL. It is quite common for the Wayback Machine to serve cached archive if last archive was captured before last 45 minutes. """ regex = r"https?://web\.archive.org/web/([0-9]{14})/http" match = re.search(regex, str(self._archive_url)) if match is None or len(match.groups()) != 1: raise ValueError( f"Can not parse timestamp from archive URL, '{self._archive_url}'." ) string_timestamp = match.group(1) timestamp = datetime.strptime(string_timestamp, "%Y%m%d%H%M%S") timestamp_unixtime = time.mktime(timestamp.timetuple()) instance_birth_time_unixtime = time.mktime(self.instance_birth_time.timetuple()) if timestamp_unixtime < instance_birth_time_unixtime: self.cached_save = True else: self.cached_save = False return timestamp def save(self) -> str: """ Calls the SavePageNow API of the Wayback Machine with required parameters and headers to save the URL. Raises MaximumSaveRetriesExceeded is maximum retries are exhausted but still we were unable to retrieve the archive from the Wayback Machine. """ self.saved_archive = None tries = 0 while True: if tries >= 1: self.sleep(tries) self.get_save_request_headers() self.saved_archive = self.archive_url_parser() if isinstance(self.saved_archive, str): self._archive_url = self.saved_archive self.timestamp() return self.saved_archive tries += 1 if tries >= self.max_tries: raise MaximumSaveRetriesExceeded( f"Tried {tries} times but failed to save " f"and retrieve the archive for {self.url}.\n" f"Response URL:\n{self.response_url}\n" f"Response Header:\n{self.headers}" ) waybackpy-3.0.6/waybackpy/utils.py000066400000000000000000000015211421412532700172160ustar00rootroot00000000000000""" Utility functions and shared variables like DEFAULT_USER_AGENT are here. """ from datetime import datetime from . import __version__ DEFAULT_USER_AGENT: str = ( f"waybackpy {__version__} - https://github.com/akamhy/waybackpy" ) def unix_timestamp_to_wayback_timestamp(unix_timestamp: int) -> str: """ Converts Unix time to Wayback Machine timestamp, Wayback Machine timestamp format is yyyyMMddhhmmss. """ return datetime.utcfromtimestamp(int(unix_timestamp)).strftime("%Y%m%d%H%M%S") def wayback_timestamp(**kwargs: int) -> str: """ Prepends zero before the year, month, day, hour and minute so that they are conformable with the YYYYMMDDhhmmss Wayback Machine timestamp format. """ return "".join( str(kwargs[key]).zfill(2) for key in ["year", "month", "day", "hour", "minute"] ) waybackpy-3.0.6/waybackpy/wrapper.py000066400000000000000000000127251421412532700175460ustar00rootroot00000000000000""" This module exists because backwards compatibility matters. Don't touch this or add any new functionality here and don't use the Url class. """ from datetime import datetime, timedelta from typing import Generator, Optional from requests.structures import CaseInsensitiveDict from .availability_api import ResponseJSON, WaybackMachineAvailabilityAPI from .cdx_api import WaybackMachineCDXServerAPI from .save_api import WaybackMachineSaveAPI from .utils import DEFAULT_USER_AGENT class Url: """ The Url class is not recommended to be used anymore, instead use: - WaybackMachineSaveAPI - WaybackMachineAvailabilityAPI - WaybackMachineCDXServerAPI The reason it is still in the code is backwards compatibility with 2.x.x versions. If were are using the Url before the update to version 3.x.x, your code should still be working fine and there is no hurry to update the interface but is recommended that you do not use the Url class for new code as it would be removed after 2025 also the first 3.x.x versions was released in January 2022 and three years are more than enough to update the older interface code. """ def __init__(self, url: str, user_agent: str = DEFAULT_USER_AGENT) -> None: self.url = url self.user_agent = str(user_agent) self.archive_url: Optional[str] = None self.timestamp: Optional[datetime] = None self.wayback_machine_availability_api = WaybackMachineAvailabilityAPI( self.url, user_agent=self.user_agent ) self.wayback_machine_save_api: Optional[WaybackMachineSaveAPI] = None self.headers: Optional[CaseInsensitiveDict[str]] = None self.json: Optional[ResponseJSON] = None def __str__(self) -> str: if not self.archive_url: self.newest() return str(self.archive_url) def __len__(self) -> int: td_max = timedelta( days=999999999, hours=23, minutes=59, seconds=59, microseconds=999999 ) if not isinstance(self.timestamp, datetime): self.oldest() if not isinstance(self.timestamp, datetime): raise TypeError("timestamp must be a datetime") if self.timestamp == datetime.max: return td_max.days return (datetime.utcnow() - self.timestamp).days def save(self) -> "Url": """Save the URL on wayback machine.""" self.wayback_machine_save_api = WaybackMachineSaveAPI( self.url, user_agent=self.user_agent ) self.archive_url = self.wayback_machine_save_api.archive_url self.timestamp = self.wayback_machine_save_api.timestamp() self.headers = self.wayback_machine_save_api.headers return self def near( self, year: Optional[int] = None, month: Optional[int] = None, day: Optional[int] = None, hour: Optional[int] = None, minute: Optional[int] = None, unix_timestamp: Optional[int] = None, ) -> "Url": """Returns the archive of the URL close to a date and time.""" self.wayback_machine_availability_api.near( year=year, month=month, day=day, hour=hour, minute=minute, unix_timestamp=unix_timestamp, ) self.set_availability_api_attrs() return self def oldest(self) -> "Url": """Returns the oldest archive of the URL.""" self.wayback_machine_availability_api.oldest() self.set_availability_api_attrs() return self def newest(self) -> "Url": """Returns the newest archive of the URL.""" self.wayback_machine_availability_api.newest() self.set_availability_api_attrs() return self def set_availability_api_attrs(self) -> None: """Set the attributes for total backwards compatibility.""" self.archive_url = self.wayback_machine_availability_api.archive_url self.json = self.wayback_machine_availability_api.json self.JSON = self.json # for backwards compatibility, do not remove it. self.timestamp = self.wayback_machine_availability_api.timestamp() def total_archives( self, start_timestamp: Optional[str] = None, end_timestamp: Optional[str] = None ) -> int: """ Returns an integer which indicates total number of archives for an URL. Useless in my opinion, only here because of backwards compatibility. """ cdx = WaybackMachineCDXServerAPI( self.url, user_agent=self.user_agent, start_timestamp=start_timestamp, end_timestamp=end_timestamp, ) count = 0 for _ in cdx.snapshots(): count = count + 1 return count def known_urls( self, subdomain: bool = False, host: bool = False, start_timestamp: Optional[str] = None, end_timestamp: Optional[str] = None, match_type: str = "prefix", ) -> Generator[str, None, None]: """Yields known URLs for any URL.""" if subdomain: match_type = "domain" if host: match_type = "host" cdx = WaybackMachineCDXServerAPI( self.url, user_agent=self.user_agent, start_timestamp=start_timestamp, end_timestamp=end_timestamp, match_type=match_type, collapses=["urlkey"], ) for snapshot in cdx.snapshots(): yield snapshot.original