pax_global_header00006660000000000000000000000064140227544310014514gustar00rootroot0000000000000052 comment=11c2ee8557c5506eb2fbd59416d9a8cf0f9df0ea cti-taxii-client-2.3.0/000077500000000000000000000000001402275443100146655ustar00rootroot00000000000000cti-taxii-client-2.3.0/.github/000077500000000000000000000000001402275443100162255ustar00rootroot00000000000000cti-taxii-client-2.3.0/.github/workflows/000077500000000000000000000000001402275443100202625ustar00rootroot00000000000000cti-taxii-client-2.3.0/.github/workflows/python-ci-tests.yml000066400000000000000000000017311402275443100240610ustar00rootroot00000000000000# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions name: cti-taxii-client test harness on: [push, pull_request] jobs: build: runs-on: ubuntu-latest strategy: matrix: python-version: [3.6, 3.7, 3.8, 3.9] name: Python ${{ matrix.python-version }} Build steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Install and update essential dependencies run: | pip install -U pip setuptools pip install tox-gh-actions pip install codecov - name: Test with Tox run: | tox - name: Upload coverage information to Codecov uses: codecov/codecov-action@v1 with: fail_ci_if_error: true # optional (default = false) verbose: true # optional (default = false) cti-taxii-client-2.3.0/.gitignore000066400000000000000000000021761402275443100166630ustar00rootroot00000000000000# Created by .ignore support plugin (hsz.mobi) ### Python template # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python env/ build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ *.egg-info/ .installed.cfg *.egg # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *,cover .hypothesis/ .pytest_cache/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # pyenv .python-version # celery beat schedule file celerybeat-schedule # dotenv .env # virtualenv .venv/ venv/ ENV/ # Spyder project settings .spyderproject # Rope project settings .ropeproject .idea/ .pypirc cti-taxii-client-2.3.0/.isort.cfg000066400000000000000000000002521402275443100165630ustar00rootroot00000000000000[settings] known_third_party= pytest, pytz, requests, responses, six, known_first_party=taxii2client force_sort_within_sections=1 multi_line_output=5 cti-taxii-client-2.3.0/.pre-commit-config.yaml000066400000000000000000000010521402275443100211440ustar00rootroot00000000000000repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v3.4.0 hooks: - id: trailing-whitespace - id: check-merge-conflict - repo: https://github.com/PyCQA/flake8 rev: 3.8.4 hooks: - id: flake8 name: Check project styling args: - --max-line-length=160 - repo: https://github.com/PyCQA/isort rev: 5.7.0 hooks: - id: isort name: Sort python imports (shows diff) args: ["-c", "--diff"] - id: isort name: Sort python imports (fixes files) cti-taxii-client-2.3.0/CHANGES.txt000066400000000000000000000056141402275443100165040ustar00rootroot00000000000000Version 2.3.0 2021-03-12 - #95 Remove extra whitespace in Content-Type header (@2xyo) - #94 Pagination changes to support more of the header combinations used in TAXII 2.0 (RFC7233). Provide similar as_pages method for v21 paginated call. - #93 Fixes to ReadTheDocs not rendering Reference API documentation - #91 Transitioned test suite from TravisCI to GitHub Workflows - #89 Expose the `cert` parameter to all API classes in v20 and v21 Version 2.2.2 2020-09-10 - #85 Provide more details when a Server Response status code is 406. (v20, v21) - #84 Fallback "Range" HTTP Header for pagination requests. (@teizenman) (v20) - #81 More flexibility for the Status Endpoint validation. It no longer fails for optional successes, pendings, failures lists (@maybe-sybr) (v20, v21) Version 2.2.1 2020-07-17 - #78 Prevent KeyError when TAXII Server Response does not include 'Content-Range' Header (v20) Version 2.2.0 2020-07-02 - #76 drop python versions older than 3.5 - #75 fixes construction of Range Header for paginated requests RFC 7233. (@dougle) Version 2.1.0 2020-06-09 - #68 Support for user-specified authentication including token-based authentication. - #71 Don't log a warning when the total number of available objects is less than the per request size limit. Version 2.0.0 2020-04-01 Base release of the TAXII Client for 2.1. Unsupported features: - TLS support in requests - Force connection initiation using HTTPS Note `import taxii2client` by default will load TAXII 2.1 classes. If this is not desired use `from taxii2client import v20` or `from taxii2client.v20 import ` Version 1.0.1 2020-04-01 - Fix packaging problem - #60, #23 Pagination generator function available ``as_pages`` for get_objects and get_manifest requests Version 1.0.0 2020-04-01 Base release of the TAXII Client for 2.0. Unsupported features: - TLS support in requests - Force connection initiation using HTTPS Version 0.5.0 2018-11-09 - #50 Fix an issue regarding trailing slash being missing in the Endpoint url - #51 Expose `proxies` to all Endpoint for better flexibility - #53 Better handling of custom content. It is now stored in the `custom_properties` dictionary Version 0.4.0 2018-10-31 - #36 Added "proxies" argument to _HTTPConnection. - Created documentation and host on ReadTheDocs. - #35 Handle invalid content better. - #39 Added "_raw" to provide access to raw TAXII Responses. - #40 Allow users to specify the user agent. Version 0.3.1 2018-04-12 - Fix packaging problem Version 0.3.0 2018-04-12 - #21 Re-work the accept parameters validation - #24 Expose verify positional parameter on endpoints - #27 Expose the accept positional parameter to allow configurable requests - #30 Do not fail hard for optional fields of a response Version 0.2.0 2017-11-07 - Filter keyword arguments to query params - Add datetime formatting - Improve README documentation - Improve code coverage/testing Version 0.1.0 2017-11-07 - Initial release cti-taxii-client-2.3.0/CONTRIBUTING.md000066400000000000000000000157401402275443100171250ustar00rootroot00000000000000

Contributing

Public Participation Invited

This OASIS TC Open Repository ( github.com/oasis-open/cti-taxii-client ) is a community public repository that supports participation by anyone, whether affiliated with OASIS or not. Substantive contributions (repository "code") and related feedback is invited from all parties, following the common conventions for participation in GitHub public repository projects. Participation is expected to be consistent with the OASIS TC Open Repository Guidelines and Procedures, the LICENSE designated for this particular repository (BSD-3-Clause License), and the requirement for an Individual Contributor License Agreement. Please see the repository README document for other details.

Governance Distinct from OASIS TC Process

Content accepted as "contributions" to this TC Open Repository, as defined below, are distinct from any Contributions made to the associated OASIS Cyber Threat Intelligence (CTI) TC itself. Participation in the associated Technical Committee is governed by the OASIS Bylaws, OASIS TC Process, IPR Policy, and related policies. This TC Open Repository is not subject to the OASIS TC-related policies. TC Open Repository governance is defined by separate participation and contribution guidelines as referenced in the OASIS TC Open Repositories Overview.

Licensing Distinct from OASIS IPR Policy

Because different licenses apply to the OASIS TC's specification work, and this TC Open Repository, there is no guarantee that the licensure of specific repository material will be compatible with licensing requirements of an implementation of a TC's specification. Please refer to the LICENSE file for the terms of this material, and to the OASIS IPR Policy for the terms applicable to the TC's specifications, including any applicable declarations.

Contributions Subject to Individual CLA

Formally, "contribution" to this TC Open Repository refers to content merged into the "Code" repository (repository changes represented by code commits), following the GitHub definition of contributor: "someone who has contributed to a project by having a pull request merged but does not have collaborator [i.e., direct write] access." Anyone who signs the TC Open Repository Individual Contributor License Agreement (CLA), signifying agreement with the licensing requirement, may contribute substantive content — subject to evaluation of a GitHub pull request. The main web page for this repository, as with any GitHub public repository, displays a link to a document listing contributions to the repository's default branch (filtered by Commits, Additions, and Deletions).

This TC Open Repository, as with GitHub public repositories generally, also accepts public feedback from any GitHub user. Public feedback includes opening issues, authoring and editing comments, participating in conversations, making wiki edits, creating repository stars, and making suggestions via pull requests. Such feedback does not constitute an OASIS TC Open Repository contribution. Some details are presented under "Read permissions" in the table of permission levels for a GitHub organization. Technical content intended as a substantive contribution (repository "Code") to an TC Open Repository is subject to evaluation, and requires a signed Individual CLA.

Fork-and-Pull Collaboration Model

OASIS TC Open Repositories use the familiar fork-and-pull collaboration model supported by GitHub and other distributed version-control systems. Any GitHub user wishing to contribute should fork the repository, make additions or other modifications, and then submit a pull request. GitHub pull requests should be accompanied by supporting comments and/or issues. Community conversations about pull requests, supported by GitHub notifications, will provide the basis for a consensus determination to merge, modify, close, or take other action, as communicated by the repository Maintainers.

Feedback

Questions or comments about this TC Open Repository's activities should be composed as GitHub issues or comments. If use of an issue/comment is not possible or appropriate, questions may be directed by email to the repository Maintainer(s). Please send general questions about TC Open Repository participation to OASIS Staff at repository-admin@oasis-open.org and any specific CLA-related questions to repository-cla@oasis-open.org.

cti-taxii-client-2.3.0/LICENSE000066400000000000000000000027221402275443100156750ustar00rootroot00000000000000Copyright (c) [2017], OASIS Open All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cti-taxii-client-2.3.0/MANIFEST.in000066400000000000000000000001121402275443100164150ustar00rootroot00000000000000include LICENSE include CHANGES.txt recursive-exclude taxii2client\test * cti-taxii-client-2.3.0/README.rst000066400000000000000000000224641402275443100163640ustar00rootroot00000000000000|Build_Status| |Coverage| |Version| cti-taxii-client ================ NOTE: This is an `OASIS TC Open Repository `_. See the `Governance`_ section for more information. cti-taxii-client is a minimal client implementation for the TAXII 2.X server. It supports the following TAXII 2.X API services: - Server Discovery - Get API Root Information - Get Status - Get Collections - Get a Collection - Get Objects - Add Objects - Get an Object - Delete an Object (2.1 only) - Get Object Manifests - Get Object Versions (2.1 only) Installation ------------ The easiest way to install the TAXII client is with pip .. code-block:: bash $ pip install taxii2-client Usage ----- The TAXII client is intended to be used as a Python library. There are no command line clients at this time. ``taxii2-client`` provides four classes: - ``Server`` - ``ApiRoot`` - ``Collection`` - ``Status`` Each can be instantiated by passing a `url`, and (optional) `user` and `password` arguments. The authorization information is stored in the instance, so it need not be supplied explicitly when requesting services. By default, the latest version of the supported spec will be imported. If you need a specific version you can perform the following: .. code-block:: python from taxii2client.v21 import Server server = Server('https://example.com/taxii2/', user='user_id', password='user_password') Once you have instantiated a ``Server`` object, you can get all metadata about its contents via its properties: .. code-block:: python print(server.title) This will lazily load and cache the server's information in the instance: - ``api_roots`` - ``title`` - ``description`` - ``default`` (i.e. the default API root) - ``contact`` You can follow references to ``ApiRoot`` objects, ``Collection`` objects, and (STIX) objects in those collections. .. code-block:: python api_root = server.api_roots[0] collection = api_root.collections[0] collection.add_objects(stix_bundle) Each ``ApiRoot`` has attributes corresponding to its meta data - ``title`` - ``description`` - ``max_content_length`` - ``collections`` Each ``Collection`` has attributes corresponding to its meta data: - ``id`` - ``title`` - ``description`` - ``alias`` (2.1 only) - ``can_write`` - ``can_read`` - ``media_types`` A ``Collection`` can also be instantiated directly: .. code-block:: python # Performing TAXII 2.0 Requests from taxii2client.v20 import Collection, as_pages collection = Collection('https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116') print(collection.get_object('indicator--252c7c11-daf2-42bd-843b-be65edca9f61')) # For normal (no pagination) requests print(collection.get_objects()) print(collection.get_manifest()) # For pagination requests. # Use *args for other arguments to the call and **kwargs to pass filter information for bundle in as_pages(collection.get_objects, per_request=50): print(bundle) for manifest_resource in as_pages(collection.get_manifest, per_request=50): print(manifest_resource) # ---------------------------------------------------------------- # # Performing TAXII 2.1 Requests from taxii2client.v21 import Collection, as_pages collection = Collection('https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116') print(collection.get_object('indicator--252c7c11-daf2-42bd-843b-be65edca9f61')) # For normal (no pagination) requests print(collection.get_objects()) print(collection.get_manifest()) # For pagination requests. # Use *args for other arguments to the call and **kwargs to pass filter information for envelope in as_pages(collection.get_objects, per_request=50): print(envelope) for manifest_resource in as_pages(collection.get_manifest, per_request=50): print(manifest_resource) In addition to the object-specific properties and methods, all classes have a ``refresh()`` method that reloads the URL corresponding to that resource, to ensure properties have the most up-to-date values. Governance ---------- This GitHub public repository ( **https://github.com/oasis-open/cti-taxii-client** ) was created at the request of the `OASIS Cyber Threat Intelligence (CTI) TC `__ as an `OASIS TC Open Repository `__ to support development of open source resources related to Technical Committee work. While this TC Open Repository remains associated with the sponsor TC, its development priorities, leadership, intellectual property terms, participation rules, and other matters of governance are `separate and distinct `__ from the OASIS TC Process and related policies. All contributions made to this TC Open Repository are subject to open source license terms expressed in the `BSD-3-Clause License `__. That license was selected as the declared `"Applicable License" `__ when the TC Open Repository was created. As documented in `"Public Participation Invited `__", contributions to this OASIS TC Open Repository are invited from all parties, whether affiliated with OASIS or not. Participants must have a GitHub account, but no fees or OASIS membership obligations are required. Participation is expected to be consistent with the `OASIS TC Open Repository Guidelines and Procedures `__, the open source `LICENSE `__ designated for this particular repository, and the requirement for an `Individual Contributor License Agreement `__ that governs intellectual property. Maintainers ----------- TC Open Repository `Maintainers `__ are responsible for oversight of this project's community development activities, including evaluation of GitHub `pull requests `__ and `preserving `__ open source principles of openness and fairness. Maintainers are recognized and trusted experts who serve to implement community goals and consensus design preferences. Initially, the associated TC members have designated one or more persons to serve as Maintainer(s); subsequently, participating community members may select additional or substitute Maintainers, per `consensus agreements `__. Current Maintainers of this TC Open Repository ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - `Chris Lenk `__; GitHub ID: https://github.com/clenk/; WWW: `MITRE Corporation `__ - `Rich Piazza `__; GitHub ID: https://github.com/rpiazza/; WWW: `MITRE Corporation `__ - `Emmanuelle Vargas-Gonzalez `__; GitHub ID: https://github.com/emmanvg/; WWW: `MITRE Corporation `__ - `Jason Keirstead `__; GitHub ID: https://github.com/JasonKeirstead; WWW: `IBM `__ About OASIS TC Open Repositories -------------------------------- - `TC Open Repositories: Overview and Resources `__ - `Frequently Asked Questions `__ - `Open Source Licenses `__ - `Contributor License Agreements (CLAs) `__ - `Maintainers' Guidelines and Agreement `__ Feedback -------- Questions or comments about this TC Open Repository's activities should be composed as GitHub issues or comments. If use of an issue/comment is not possible or appropriate, questions may be directed by email to the Maintainer(s) `listed above <#currentMaintainers>`__. Please send general questions about Open Repository participation to OASIS Staff at repository-admin@oasis-open.org and any specific CLA-related questions to repository-cla@oasis-open.org. .. |Build_Status| image:: https://github.com/oasis-open/cti-taxii-client/workflows/cti-taxii-client%20test%20harness/badge.svg :target: https://github.com/oasis-open/cti-taxii-client/actions?query=workflow%3A%22cti-taxii-client+test+harness%22 .. |Coverage| image:: https://codecov.io/gh/oasis-open/cti-taxii-client/branch/master/graph/badge.svg :target: https://codecov.io/gh/oasis-open/cti-taxii-client .. |Version| image:: https://img.shields.io/pypi/v/taxii2-client.svg?maxAge=3600 :target: https://pypi.python.org/pypi/taxii2-client/ cti-taxii-client-2.3.0/dev-requirements.txt000066400000000000000000000000441402275443100207230ustar00rootroot00000000000000bumpversion==0.5.3 -e .[docs,test] cti-taxii-client-2.3.0/docs/000077500000000000000000000000001402275443100156155ustar00rootroot00000000000000cti-taxii-client-2.3.0/docs/Makefile000066400000000000000000000011421402275443100172530ustar00rootroot00000000000000# Minimal makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build SPHINXPROJ = taxii2client SOURCEDIR = . BUILDDIR = _build # Put it first so that "make" without argument is like "make help". help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) .PHONY: help Makefile # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) cti-taxii-client-2.3.0/docs/api/000077500000000000000000000000001402275443100163665ustar00rootroot00000000000000cti-taxii-client-2.3.0/docs/api/taxii2client.rst000066400000000000000000000015501402275443100215200ustar00rootroot00000000000000API Reference ============= This section of documentation contains information on all of the classes and functions in the ``taxii2client`` API, as given by the package's docstrings. Subpackages ----------- .. toctree:: :maxdepth: 4 taxii2client.v20 taxii2client.v21 Submodules ---------- taxii2client.common module ************************** .. automodule:: taxii2client.common :members: :undoc-members: :show-inheritance: taxii2client.exceptions module ****************************** .. automodule:: taxii2client.exceptions :members: :undoc-members: :show-inheritance: taxii2client.version module *************************** .. automodule:: taxii2client.version :members: :undoc-members: :show-inheritance: Module contents --------------- .. automodule:: taxii2client :members: :undoc-members: :show-inheritance: cti-taxii-client-2.3.0/docs/api/taxii2client.v20.rst000066400000000000000000000002531402275443100221250ustar00rootroot00000000000000taxii2client.v20 package ======================== Module contents --------------- .. automodule:: taxii2client.v20 :members: :undoc-members: :show-inheritance: cti-taxii-client-2.3.0/docs/api/taxii2client.v21.rst000066400000000000000000000002531402275443100221260ustar00rootroot00000000000000taxii2client.v21 package ======================== Module contents --------------- .. automodule:: taxii2client.v21 :members: :undoc-members: :show-inheritance: cti-taxii-client-2.3.0/docs/conf.py000066400000000000000000000122641402275443100171210ustar00rootroot00000000000000#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # taxii2client documentation build configuration file, created by # sphinx-quickstart on Tue Nov 28 20:47:27 2017. # # This file is execfile()d with the current directory set to its # containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # # import os # import sys # sys.path.insert(0, os.path.abspath('.')) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. # # needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ 'sphinx-prompt', 'sphinx.ext.autodoc', 'sphinx.ext.autosummary', 'sphinx.ext.napoleon', 'sphinx.ext.todo', ] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] source_suffix = '.rst' # The master toctree document. master_doc = 'index' # General information about the project. project = 'taxii2client' copyright = '2017, OASIS Open' author = 'OASIS Open' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = '2.3.0' # The full version, including alpha/beta/rc tags. release = '2.3.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. language = None # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # html_theme = 'alabaster' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # # html_theme_options = {} # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] # Custom sidebar templates, must be a dictionary that maps document names # to template names. # # This is required for the alabaster theme # refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars html_sidebars = { '**': [ 'about.html', 'navigation.html', 'relations.html', # needs 'show_related': True theme option to display 'searchbox.html', ] } # -- Options for HTMLHelp output ------------------------------------------ # Output file base name for HTML help builder. htmlhelp_basename = 'taxii2clientdoc' # -- Options for LaTeX output --------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', # Additional stuff for the LaTeX preamble. # # 'preamble': '', # Latex figure (float) alignment # # 'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ (master_doc, 'taxii2client.tex', 'taxii2client Documentation', 'OASIS Open', 'manual'), ] # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ (master_doc, 'taxii2client', 'taxii2client Documentation', [author], 1) ] # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ (master_doc, 'taxii2client', 'taxii2client Documentation', author, 'taxii2client', 'One line description of project.', 'Miscellaneous'), ] cti-taxii-client-2.3.0/docs/contributing.rst000066400000000000000000000077701402275443100210710ustar00rootroot00000000000000Contributing ============ We're thrilled that you're interested in contributing to the taxii-client! Here are some things you should know: - `contribution-guide.org `_ has great ideas for contributing to any open-source project (not just this one). - All contributors must sign a Contributor License Agreement. See `CONTRIBUTING.md `_ in the project repository for specifics. - If you are planning to implement a major feature (vs. fixing a bug), please discuss with a project maintainer first to ensure you aren't duplicating the work of someone else, and that the feature is likely to be accepted. Now, let's get started! Setting up a development environment ------------------------------------ We recommend using a `virtualenv `_. 1. Clone the repository. If you're planning to make pull request, you should fork the repository on GitHub and clone your fork instead of the main repo: .. code-block:: bash $ git clone https://github.com/yourusername/cti-taxii-client.git 2. Install develoment-related dependencies: .. code-block:: bash $ cd cti-taxii-client $ pip install -r requirements.txt 3. Install `pre-commit `_ git hooks: .. code-block:: bash $ pre-commit install At this point you should be able to make changes to the code. Code style ---------- All code should follow `PEP 8 `_. We allow for line lengths up to 160 characters, but any lines over 80 characters should be the exception rather than the rule. PEP 8 conformance will be tested automatically by Tox and Travis-CI (see below). Testing ------- .. note:: All of the tools mentioned in this section are installed when you run ``pip install -r requirements.txt``. This project uses `pytest `_ for testing. We encourage the use of test-driven development (TDD), where you write (failing) tests that demonstrate a bug or proposed new feature before writing code that fixes the bug or implements the features. Any code contributions should come with new or updated tests. To run the tests in your current Python environment, use the ``pytest`` command from the root project directory: .. code-block:: bash $ pytest This should show all of the tests that ran, along with their status. You can run a specific test file by passing it on the command line: .. code-block:: bash $ pytest taxii2client/test/test_.py To ensure that the test you wrote is running, you can deliberately add an ``assert False`` statement at the beginning of the test. This is another benefit of TDD, since you should be able to see the test failing (and ensure it's being run) before making it pass. `tox `_ allows you to test a package across multiple versions of Python. Setting up multiple Python environments is beyond the scope of this guide, but feel free to ask for help setting them up. Tox should be run from the root directory of the project: .. code-block:: bash tox We aim for high test coverage, using the `coverage.py `_ library. Though it's not an absolute requirement to maintain 100% coverage, all code contributions must be accompanied by tests. To run coverage and look for untested lines of code, run: .. code-block:: bash $ pytest --cov=taxii2client $ coverage html then look at the resulting report in ``htmlcov/index.html``. All commits pushed to the ``master`` branch or submitted as a pull request are tested with `Travis-CI `_ automatically. Adding a dependency ------------------- One of the pre-commit hooks we use in our develoment environment enforces a consistent ordering to imports. If you need to add a new library as a dependency please add it to the `known_third_party` section of `.isort.cfg` to make sure the import is sorted correctly. cti-taxii-client-2.3.0/docs/index.rst000066400000000000000000000015111402275443100174540ustar00rootroot00000000000000.. taxii2client documentation master file, created by sphinx-quickstart on Tue Nov 28 20:47:27 2017. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. TAXII 2 Python API Documentation ================================ *taxii2client* is a minimal implementation of a TAXII 2.0 and 2.1 Client in Python. It currently supports the current services: - Server Discovery - Get API Root Information - Get Status - Get Collections - Get a Collection - Get Objects - Add Objects - Get an Object - Delete an Object (2.1 only) - Get Object Manifests - Get Object Versions (2.1 only) Table of Contents ================= .. toctree:: :maxdepth: 3 api/taxii2client contributing Indices and tables ================== * :ref:`genindex` * :ref:`modindex` * :ref:`search` cti-taxii-client-2.3.0/setup.cfg000066400000000000000000000003721402275443100165100ustar00rootroot00000000000000[bumpversion] current_version = 2.3.0 commit = True tag = True [bumpversion:file:docs/conf.py] [bumpversion:file:taxii2client/version.py] [bumpversion:file:taxii2client/__init__.py] [metadata] license_file = LICENSE [bdist_wheel] universal = 1 cti-taxii-client-2.3.0/setup.py000066400000000000000000000042511402275443100164010ustar00rootroot00000000000000#!/usr/bin/env python from codecs import open import os from setuptools import find_packages, setup BASE_DIR = os.path.dirname(os.path.abspath(__file__)) VERSION_FILE = os.path.join(BASE_DIR, 'taxii2client', 'version.py') def get_version(): with open(VERSION_FILE) as f: for line in f.readlines(): if line.startswith('__version__'): version = line.split()[-1].strip('"') return version raise AttributeError("Package does not have a __version__") def get_long_description(): with open('README.rst') as f: return f.read() setup( name='taxii2-client', version=get_version(), description='TAXII 2 Client Library', long_description=get_long_description(), long_description_content_type='text/x-rst', url='https://oasis-open.github.io/cti-documentation/', author='OASIS Cyber Threat Intelligence Technical Committee', author_email='cti-users@lists.oasis-open.org', maintainer='Emmanuelle Vargas-Gonzalez', maintainer_email='emmanuelle@mitre.org', license='BSD', classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', 'Topic :: Security', 'License :: OSI Approved :: BSD License', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', ], keywords='taxii taxii2 client json cti cyber threat intelligence', packages=find_packages(exclude=['*.test']), install_requires=[ 'requests', 'six', 'pytz', ], extras_require={ 'test': [ 'coverage', 'pytest', 'pytest-cov', 'responses', 'tox', ], 'docs': [ 'sphinx', 'sphinx-prompt', ] }, project_urls={ 'Documentation': 'https://taxii2client.readthedocs.io/', 'Source Code': 'https://github.com/oasis-open/cti-taxii-client/', 'Bug Tracker': 'https://github.com/oasis-open/cti-taxii-client/issues/', }, ) cti-taxii-client-2.3.0/taxii2client/000077500000000000000000000000001402275443100172645ustar00rootroot00000000000000cti-taxii-client-2.3.0/taxii2client/__init__.py000066400000000000000000000012741402275443100214010ustar00rootroot00000000000000"""Python TAXII 2 Client""" # flake8: noqa # isort:skip_file import logging # Console Handler for taxii2client messages ch = logging.StreamHandler() ch.setFormatter(logging.Formatter("[%(name)s] [%(levelname)-8s] [%(asctime)s] %(message)s")) # Module-level logger log = logging.getLogger(__name__) log.propagate = False log.addHandler(ch) DEFAULT_USER_AGENT = "taxii2-client/2.3.0" MEDIA_TYPE_STIX_V20 = "application/vnd.oasis.stix+json; version=2.0" MEDIA_TYPE_TAXII_V20 = "application/vnd.oasis.taxii+json; version=2.0" MEDIA_TYPE_TAXII_V21 = "application/taxii+json;version=2.1" from .v21 import * # This import will always be the latest TAXII 2.X version from .version import __version__ cti-taxii-client-2.3.0/taxii2client/common.py000066400000000000000000000363541402275443100211410ustar00rootroot00000000000000import datetime import logging import pytz import requests import requests.auth import requests.structures import six from . import DEFAULT_USER_AGENT, MEDIA_TYPE_TAXII_V20, MEDIA_TYPE_TAXII_V21 from .exceptions import ( InvalidArgumentsError, InvalidJSONError, TAXIIServiceException ) # Module-level logger log = logging.getLogger(__name__) def _format_datetime(dttm): """Convert a datetime object into a valid STIX timestamp string. 1. Convert to timezone-aware 2. Convert to UTC 3. Format in ISO format 4. Ensure correct precision a. Add subsecond value if non-zero and precision not defined 5. Add "Z" """ if dttm.tzinfo is None or dttm.tzinfo.utcoffset(dttm) is None: # dttm is timezone-naive; assume UTC zoned = pytz.utc.localize(dttm) else: zoned = dttm.astimezone(pytz.utc) ts = zoned.strftime("%Y-%m-%dT%H:%M:%S") ms = zoned.strftime("%f") precision = getattr(dttm, "precision", None) if precision == "second": pass # Already precise to the second elif precision == "millisecond": ts = ts + "." + ms[:3] elif zoned.microsecond > 0: ts = ts + "." + ms.rstrip("0") return ts + "Z" def _ensure_datetime_to_string(maybe_dttm): """If maybe_dttm is a datetime instance, convert to a STIX-compliant string representation. Otherwise return the value unchanged.""" if isinstance(maybe_dttm, datetime.datetime): maybe_dttm = _format_datetime(maybe_dttm) return maybe_dttm def _filter_kwargs_to_query_params(filter_kwargs): """ Convert API keyword args to a mapping of URL query parameters. Except for "added_after", all keywords are mapped to match filters, i.e. to a query parameter of the form "match[]". "added_after" is left alone, since it's a special filter, as defined in the spec. Each value can be a single value or iterable of values. "version" and "added_after" get special treatment, since they are timestamp-valued: datetime.datetime instances are supported and automatically converted to STIX-compliant strings. Other than that, all values must be strings. None values, empty lists, etc are silently ignored. Args: filter_kwargs: The filter information, as a mapping. Returns: query_params (dict): The query parameter map, mapping strings to strings. """ query_params = {} for kwarg, arglist in six.iteritems(filter_kwargs): # If user passes an empty list, None, etc, silently skip? if not arglist: continue # force iterability, for the sake of code uniformity if not hasattr(arglist, "__iter__") or \ isinstance(arglist, six.string_types): arglist = arglist, if kwarg == "version": query_params["match[version]"] = ",".join( _ensure_datetime_to_string(val) for val in arglist ) elif kwarg == "added_after": if len(arglist) > 1: raise InvalidArgumentsError("No more than one value for filter" " 'added_after' may be given") query_params["added_after"] = ",".join( _ensure_datetime_to_string(val) for val in arglist ) elif kwarg == "limit": query_params["limit"] = int(arglist[0]) elif kwarg == "next": query_params["next"] = arglist else: query_params["match[" + kwarg + "]"] = ",".join(arglist) return query_params def _to_json(resp): """ Factors out some JSON parse code with error handling, to hopefully improve error messages. :param resp: A requests.Response instance :return: Parsed JSON. :raises: InvalidJSONError If JSON parsing failed. """ try: return resp.json() except ValueError as e: # Maybe better to report the original request URL? six.raise_from(InvalidJSONError( "Invalid JSON was received from " + resp.request.url ), e) def _grab_total_items_from_resource(resp): """Returns number of objects in bundle/envelope""" if isinstance(resp, requests.Response): resp = _to_json(resp) return len(resp.get("objects", [])) class TokenAuth(requests.auth.AuthBase): def __init__(self, key): self.key = key def __call__(self, r): r.headers['Authorization'] = 'Token {}'.format(self.key) return r class _TAXIIEndpoint(object): """Contains some data and functionality common to all TAXII endpoint classes: a URL, connection, and ability to close the connection. It also yields support in subclasses for use as context managers, to ensure resources are released. """ def __init__(self, url, conn=None, user=None, password=None, verify=True, proxies=None, version="2.0", auth=None, cert=None): """Create a TAXII endpoint. Args: user (str): username for authentication (optional) password (str): password for authentication (optional) verify (bool): validate the entity credentials (default: True) conn (_HTTPConnection): A connection to reuse (optional) proxies (dict): key/value pair for http/https proxy settings. (optional) version (str): The spec version this connection is meant to follow. cert (str or tuple): SSL client certificate default, if String, path to ssl client cert file (.pem). If Tuple, (‘cert’, ‘key’) pair. (optional) """ if (conn and ((user or password) or auth)) or ((user or password) and auth): raise InvalidArgumentsError("Only one of a connection, username/password, or auth object may" " be provided.") elif conn: self._conn = conn else: self._conn = _HTTPConnection(user, password, verify, proxies, version=version, auth=auth, cert=cert) # Add trailing slash to TAXII endpoint if missing # https://github.com/oasis-open/cti-taxii-client/issues/50 if url[-1] == "/": self.url = url else: self.url = url + "/" def close(self): self._conn.close() def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() return False class _HTTPConnection(object): """This library uses the ``requests`` library, which presents a convenience API which hides many network details like actual connection objects. So this class doesn't represent a traditional ``connection`` either. It's a sort of approximation: sets of connections (or connection pools) and common metadata for a particular server interaction. You can send requests to any hosts via the same instance; hosts/ports are not checked and new connection pools pop into existence as needed, but all connections are closed when the close() method is called. So this is intended to be used for an independent self-contained interaction. Attributes: session (requests.Session): A requests session object. """ def __init__(self, user=None, password=None, verify=True, proxies=None, user_agent=DEFAULT_USER_AGENT, version="2.0", auth=None, cert=None): """Create a connection session. Args: user (str): username for authentication (optional) password (str): password for authentication (optional) verify (bool): validate the entity credentials. (default: True) proxies (dict): key/value pair for http/https proxy settings. (optional) user_agent (str): A value to use for the User-Agent header in requests. If not given, use a default value which represents this library. version (str): The spec version this connection is meant to follow. cert (str or tuple): SSL client certificate default, if String, path to ssl client cert file (.pem). If Tuple, (‘cert’, ‘key’) pair. (optional) """ self.session = requests.Session() self.session.verify = verify # enforce that we always have a connection-default user agent. self.user_agent = user_agent or DEFAULT_USER_AGENT if user and password: self.session.auth = requests.auth.HTTPBasicAuth(user, password) elif auth: self.session.auth = auth if proxies: self.session.proxies.update(proxies) self.version = version if cert: self.session.cert = cert def valid_content_type(self, content_type, accept): """Check that the server is returning a valid Content-Type Args: content_type (str): ``Content-Type:`` header value accept (str): media type to include in the ``Accept:`` header. """ accept_tokens = accept.replace(' ', '').split(';') content_type_tokens = content_type.replace(' ', '').split(';') if self.version == "2.0": return ( all(elem in content_type_tokens for elem in accept_tokens) and (content_type_tokens[0] == 'application/vnd.oasis.taxii+json' or content_type_tokens[0] == 'application/vnd.oasis.stix+json') ) else: return ( all(elem in content_type_tokens for elem in accept_tokens) and content_type_tokens[0] == 'application/taxii+json' ) def get(self, url, headers=None, params=None): """Perform an HTTP GET, using the saved requests.Session and auth info. If "Accept" isn't one of the given headers, a default TAXII mime type is used. Regardless, the response type is checked against the accept header value, and an exception is raised if they don't match. Args: url (str): URL to retrieve headers (dict): Any other headers to be added to the request. params: dictionary or bytes to be sent in the query string for the request. (optional) """ merged_headers = self._merge_headers(headers) if self.version == "2.0": media_type = MEDIA_TYPE_TAXII_V20 else: media_type = MEDIA_TYPE_TAXII_V21 if "Accept" not in merged_headers: merged_headers["Accept"] = media_type accept = merged_headers["Accept"] resp = self.session.get(url, headers=merged_headers, params=params) try: resp.raise_for_status() except requests.exceptions.HTTPError as e: if resp.status_code == 406: # Provide more details about this error since its usually an import problem. # Import the correct version of the TAXII Client. logging.error( "Server Response: 406 Client Error " "If you are trying to contact a TAXII 2.0 Server use 'from taxii2client.v20 import X'. " "If you are trying to contact a TAXII 2.1 Server use 'from taxii2client.v21 import X'" ) raise e content_type = resp.headers["Content-Type"] if not self.valid_content_type(content_type=content_type, accept=accept): msg = ( "Unexpected Response. Got Content-Type: '{}' for Accept: '{}'\n" "If you are trying to contact a TAXII 2.0 Server use 'from taxii2client.v20 import X'\n" "If you are trying to contact a TAXII 2.1 Server use 'from taxii2client.v21 import X'" ) raise TAXIIServiceException(msg.format(content_type, accept)) if "Range" in merged_headers and self.version == "2.0": return resp else: return _to_json(resp) def post(self, url, headers=None, params=None, **kwargs): """Send a JSON POST request with the given request headers, additional URL query parameters, and the given JSON in the request body. The extra query parameters are merged with any which already exist in the URL. The 'json' and 'data' parameters may not both be given. Args: url (str): URL to retrieve headers (dict): Any other headers to be added to the request. params: dictionary or bytes to be sent in the query string for the request. (optional) json: json to send in the body of the Request. This must be a JSON-serializable object. (optional) data: raw request body data. May be a dictionary, list of tuples, bytes, or file-like object to send in the body of the Request. (optional) """ if len(kwargs) > 1: raise InvalidArgumentsError("Too many extra args ({} > 1)".format( len(kwargs))) if kwargs: kwarg = next(iter(kwargs)) if kwarg not in ("json", "data"): raise InvalidArgumentsError("Invalid kwarg: " + kwarg) resp = self.session.post(url, headers=headers, params=params, **kwargs) resp.raise_for_status() return _to_json(resp) def delete(self, url, headers=None, params=None, **kwargs): """Perform HTTP DELETE""" # TODO: May need more work... resp = self.session.delete(url, headers=headers, params=params, **kwargs) resp.raise_for_status() def close(self): """Closes connections. This object is no longer usable.""" self.session.close() def _merge_headers(self, call_specific_headers): """ Merge headers from different sources together. Headers passed to the post/get methods have highest priority, then headers associated with the connection object itself have next priority. :param call_specific_headers: A header dict from the get/post call, or None (the default for those methods). :return: A key-case-insensitive MutableMapping object which contains the merged headers. (This doesn't actually return a dict.) """ # A case-insensitive mapping is necessary here so that there is # predictable behavior. If a plain dict were used, you'd get keys in # the merged dict which differ only in case. The requests library # would merge them internally, and it would be unpredictable which key # is chosen for the final set of headers. Another possible approach # would be to upper/lower-case everything, but this seemed easier. On # the other hand, I don't know if CaseInsensitiveDict is public API...? # First establish defaults merged_headers = requests.structures.CaseInsensitiveDict({ "User-Agent": self.user_agent }) # Then overlay with specifics from post/get methods if call_specific_headers: merged_headers.update(call_specific_headers) # Special "User-Agent" header check, to ensure one is always sent. # The call-specific overlay could have null'd out that header. if not merged_headers.get("User-Agent"): merged_headers["User-Agent"] = self.user_agent return merged_headers cti-taxii-client-2.3.0/taxii2client/exceptions.py000066400000000000000000000012161402275443100220170ustar00rootroot00000000000000"""TAXII2 Error Classes.""" class TAXIIServiceException(Exception): """Base class for exceptions raised by this library.""" pass class InvalidArgumentsError(TAXIIServiceException): """Invalid arguments were passed to a method.""" pass class AccessError(TAXIIServiceException): """Attempt was made to read/write to a collection when the collection doesn't allow that operation.""" pass class ValidationError(TAXIIServiceException): """Data validation failed for a property or group of properties""" pass class InvalidJSONError(TAXIIServiceException): """A server endpoint gave us invalid JSON""" pass cti-taxii-client-2.3.0/taxii2client/test/000077500000000000000000000000001402275443100202435ustar00rootroot00000000000000cti-taxii-client-2.3.0/taxii2client/test/__init__.py000066400000000000000000000000001402275443100223420ustar00rootroot00000000000000cti-taxii-client-2.3.0/taxii2client/test/test_client_v20.py000066400000000000000000001405531402275443100236310ustar00rootroot00000000000000import datetime import json import pytest import requests import responses import six from taxii2client import ( DEFAULT_USER_AGENT, MEDIA_TYPE_STIX_V20, MEDIA_TYPE_TAXII_V20 ) from taxii2client.common import ( TokenAuth, _filter_kwargs_to_query_params, _HTTPConnection, _TAXIIEndpoint ) from taxii2client.exceptions import ( AccessError, InvalidArgumentsError, InvalidJSONError, TAXIIServiceException, ValidationError ) from taxii2client.v20 import ApiRoot, Collection, Server, Status, as_pages TAXII_SERVER = "example.com" DISCOVERY_URL = "https://{}/taxii/".format(TAXII_SERVER) API_ROOT_URL = "https://{}/api1/".format(TAXII_SERVER) COLLECTIONS_URL = API_ROOT_URL + "collections/" COLLECTION_URL = COLLECTIONS_URL + "91a7b528-80eb-42ed-a74d-c6fbd5a26116/" OBJECTS_URL = COLLECTION_URL + "objects/" GET_OBJECTS_URL = OBJECTS_URL ADD_OBJECTS_URL = OBJECTS_URL WRITABLE_COLLECTION_URL = COLLECTIONS_URL + "e278b87e-0f9b-4c63-a34c-c8f0b3e91acb/" ADD_WRITABLE_OBJECTS_URL = WRITABLE_COLLECTION_URL + "objects/" GET_OBJECT_URL = OBJECTS_URL + "indicator--252c7c11-daf2-42bd-843b-be65edca9f61/" MANIFEST_URL = COLLECTION_URL + "manifest/" STATUS_ID = "2d086da7-4bdc-4f91-900e-d77486753710" STATUS_URL = API_ROOT_URL + "status/" + STATUS_ID + "/" # These responses are provided as examples in the TAXII 2.0 specification. DISCOVERY_RESPONSE = """{ "title": "Some TAXII Server", "description": "This TAXII Server contains a listing of...", "contact": "string containing contact information", "default": "https://example.com/api2/", "api_roots": [ "https://example.com/api1/", "https://example.com/api2/", "https://example.net/trustgroup1/" ] }""" API_ROOT_RESPONSE = """{ "title": "Malware Research Group", "description": "A trust group setup for malware researchers", "versions": ["taxii-2.0"], "max_content_length": 9765625 }""" COLLECTIONS_RESPONSE = """{ "collections": [ { "id": "91a7b528-80eb-42ed-a74d-c6fbd5a26116", "title": "High Value Indicator Collection", "description": "This data collection is for collecting high value IOCs", "can_read": true, "can_write": false, "media_types": [ "application/vnd.oasis.stix+json; version=2.0" ] }, { "id": "52892447-4d7e-4f70-b94d-d7f22742ff63", "title": "Indicators from the past 24-hours", "description": "This data collection is for collecting current IOCs", "can_read": true, "can_write": false, "media_types": [ "application/vnd.oasis.stix+json; version=2.0" ] } ] }""" COLLECTION_RESPONSE = """{ "id": "91a7b528-80eb-42ed-a74d-c6fbd5a26116", "title": "High Value Indicator Collection", "description": "This data collection is for collecting high value IOCs", "can_read": true, "can_write": false, "media_types": [ "application/vnd.oasis.stix+json; version=2.0" ] }""" # This collection is not in the spec. WRITABLE_COLLECTION = """{ "id": "e278b87e-0f9b-4c63-a34c-c8f0b3e91acb", "title": "Writable Collection", "description": "This collection is a dropbox for submitting indicators", "can_read": false, "can_write": true, "media_types": [ "application/vnd.oasis.stix+json; version=2.0" ] }""" STIX_OBJECT = """ { "type": "indicator", "id": "indicator--252c7c11-daf2-42bd-843b-be65edca9f61", "created": "2016-04-06T20:03:48.000Z", "modified": "2016-04-06T20:03:48.000Z", "pattern": "[ file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e' ]", "valid_from": "2016-01-01T00:00:00Z" } """ # This bundle is used as the response to get_objects(), and also the bundle # POST'ed with add_objects(). STIX_BUNDLE = f"""{{ "type": "bundle", "id": "bundle--5d0092c5-5f74-4287-9642-33f4c354e56d", "spec_version": "2.0", "objects": [ {STIX_OBJECT} ] }}""" GET_OBJECTS_RESPONSE = STIX_BUNDLE # get_object() still returns a bundle. In this case, the bundle has only one # object (the correct one.) GET_OBJECT_RESPONSE = GET_OBJECTS_RESPONSE # This is the expected response when calling ADD_OBJECTS with the STIX_BUNDLE # above. There is only one object, and it was added successfully. This response # is not in the spec. ADD_OBJECTS_RESPONSE = """{ "id": "350dae03-d2d8-4bd3-bc1d-8160589693e3", "status": "complete", "request_timestamp": "2016-11-02T12:34:34.12345Z", "total_count": 1, "success_count": 1, "successes": [ "indicator--252c7c11-daf2-42bd-843b-be65edca9f61" ], "failure_count": 0, "pending_count": 0 }""" # This is the response in Section 5.4 of the spec. It implies a larger # bundle than what is provided in the example. ADD_OBJECTS_RESPONSE_FROM_SPEC = """{ "id": "2d086da7-4bdc-4f91-900e-d77486753710", "status": "pending", "request_timestamp": "2016-11-02T12:34:34.12345Z", "total_count": 4, "success_count": 1, "successes": [ "indicator--c410e480-e42b-47d1-9476-85307c12bcbf" ], "failure_count": 0, "pending_count": 3 }""" GET_MANIFEST_RESPONSE = """{ "objects": [ { "id": "indicator--29aba82c-5393-42a8-9edb-6a2cb1df070b", "date_added": "2016-11-01T03:04:05Z", "versions": ["2016-11-03T12:30:59.000Z","2016-12-03T12:30:59.000Z"], "media_types": ["application/vnd.oasis.stix+json; version=2.0"] }, { "id": "indicator--ef0b28e1-308c-4a30-8770-9b4851b260a5", "date_added": "2016-11-01T10:29:05Z", "versions": ["2016-11-03T12:30:59.000Z"], "media_types": ["application/vnd.oasis.stix+json; version=2.0"] } ] }""" STATUS_RESPONSE = """{ "id": "2d086da7-4bdc-4f91-900e-d77486753710", "status": "pending", "request_timestamp": "2016-11-02T12:34:34.12345Z", "total_count": 4, "success_count": 1, "successes": [ "indicator--c410e480-e42b-47d1-9476-85307c12bcbf" ], "failure_count": 1, "failures": [ { "id": "malware--664fa29d-bf65-4f28-a667-bdb76f29ec98", "message": "Unable to process object" } ], "pending_count": 2, "pendings": [ "indicator--252c7c11-daf2-42bd-843b-be65edca9f61", "relationship--045585ad-a22f-4333-af33-bfd503a683b5" ] }""" BAD_DISCOVERY_RESPONSE = """{"title":""" ERROR_MESSAGE = """{ "title": "Error condition XYZ", "description": "This error is caused when the application tries to access data...", "error_id": "1234", "error_code": "581234", "http_status": "%s", "external_details": "http://example.com/ticketnumber1/errorid-1234", "details": { "somekey1": "somevalue", "somekey2": "some other value" } }""" @pytest.fixture def status_dict(): return { "id": "2d086da7-4bdc-4f91-900e-d77486753710", "status": "pending", "request_timestamp": "2016-11-02T12:34:34.12345Z", "total_count": 4, "success_count": 1, "successes": [ "indicator--c410e480-e42b-47d1-9476-85307c12bcbf" ], "failure_count": 1, "failures": [ { "id": "malware--664fa29d-bf65-4f28-a667-bdb76f29ec98", "message": "Unable to process object" } ], "pending_count": 2, "pendings": [ "indicator--252c7c11-daf2-42bd-843b-be65edca9f61", "relationship--045585ad-a22f-4333-af33-bfd503a683b5" ] } @pytest.fixture def collection_dict(): return { "id": "e278b87e-0f9b-4c63-a34c-c8f0b3e91acb", "title": "Writable Collection", "description": "This collection is a dropbox for submitting indicators", "can_read": False, "can_write": True, "media_types": [ "application/vnd.oasis.stix+json; version=2.0" ] } @pytest.fixture def server(): """Default server object for example.com""" return Server(DISCOVERY_URL, user="foo", password="bar") @pytest.fixture def api_root(): """Default API Root object""" return ApiRoot(API_ROOT_URL) @pytest.fixture def collection(): """Default Collection object""" # The collection response is needed to get information about the collection set_collection_response() return Collection(COLLECTION_URL) @pytest.fixture def writable_collection(): """Collection with 'can_write' set to 'true'.""" set_collection_response(WRITABLE_COLLECTION_URL, WRITABLE_COLLECTION) return Collection(WRITABLE_COLLECTION_URL) @pytest.fixture def bad_writable_collection(): """Collection with 'can_write=true', but the COLLECTION_URL is different from the one in the response""" set_collection_response(response=WRITABLE_COLLECTION) return Collection(COLLECTION_URL) def set_api_root_response(response): responses.add(responses.GET, API_ROOT_URL, body=response, status=200, content_type=MEDIA_TYPE_TAXII_V20) def set_discovery_response(response): responses.add(responses.GET, DISCOVERY_URL, body=response, status=200, content_type=MEDIA_TYPE_TAXII_V20) def set_collections_response(): responses.add(responses.GET, COLLECTIONS_URL, COLLECTIONS_RESPONSE, status=200, content_type=MEDIA_TYPE_TAXII_V20) def set_collection_response(url=COLLECTION_URL, response=COLLECTION_RESPONSE): responses.add(responses.GET, url, response, status=200, content_type=MEDIA_TYPE_TAXII_V20) def set_status_response(): responses.add(responses.GET, STATUS_URL, STATUS_RESPONSE, status=200, content_type=MEDIA_TYPE_TAXII_V20) @responses.activate def test_server_discovery(server): set_discovery_response(DISCOVERY_RESPONSE) assert server._loaded is False assert server.title == "Some TAXII Server" assert server._loaded is True assert server.description == "This TAXII Server contains a listing of..." assert server.contact == "string containing contact information" assert len(server.api_roots) == 3 assert server.default is not None assert server.api_roots[1] == server.default api_root = server.api_roots[0] assert api_root.url == API_ROOT_URL assert api_root._loaded_information is False assert api_root._loaded_collections is False discovery_dict = json.loads(DISCOVERY_RESPONSE) assert server._raw == discovery_dict @responses.activate def test_bad_json_response(server): set_discovery_response(BAD_DISCOVERY_RESPONSE) with pytest.raises(InvalidJSONError): # Just do something to trigger a request server.title @responses.activate def test_minimal_discovery_response(server): # `title` is the only required field on a Discovery Response set_discovery_response('{"title": "Some TAXII Server"}') assert server.title == "Some TAXII Server" assert server.description is None assert server.contact is None assert server.api_roots == [] assert server.default is None @responses.activate def test_discovery_with_no_default(server): response = """{ "title": "Some TAXII Server", "description": "This TAXII Server contains a listing of...", "contact": "string containing contact information", "api_roots": [ "https://example.com/api1/", "https://example.com/api2/", "https://example.net/trustgroup1/" ] }""" set_discovery_response(response) assert len(server.api_roots) == 3 assert server.default is None @responses.activate def test_discovery_with_no_title(server): response = """{ "description": "This TAXII Server contains a listing of...", "contact": "string containing contact information", "api_roots": [ "https://example.com/api1/", "https://example.com/api2/", "https://example.net/trustgroup1/" ] }""" set_discovery_response(response) with pytest.raises(ValidationError) as excinfo: server.refresh() assert "No 'title' in Server Discovery for request 'https://example.com/taxii/'" == str(excinfo.value) @responses.activate def test_api_root_no_title(api_root): set_api_root_response("""{ "description": "A trust group setup for malware researchers", "versions": ["taxii-2.0"], "max_content_length": 9765625 }""") with pytest.raises(ValidationError) as excinfo: assert api_root._loaded_information is False api_root.refresh_information() assert "No 'title' in API Root for request 'https://example.com/api1/'" == str(excinfo.value) @responses.activate def test_api_root_no_versions(api_root): set_api_root_response("""{ "title": "Malware Research Group", "description": "A trust group setup for malware researchers", "max_content_length": 9765625 }""") with pytest.raises(ValidationError) as excinfo: assert api_root._loaded_information is False api_root.refresh_information() assert "No 'versions' in API Root for request 'https://example.com/api1/'" == str(excinfo.value) @responses.activate def test_api_root_no_max_content_length(api_root): set_api_root_response("""{ "title": "Malware Research Group", "description": "A trust group setup for malware researchers", "versions": ["taxii-2.0"] }""") with pytest.raises(ValidationError) as excinfo: assert api_root._loaded_information is False api_root.refresh_information() assert "No 'max_content_length' in API Root for request 'https://example.com/api1/'" == str(excinfo.value) @responses.activate def test_api_root(api_root): set_api_root_response(API_ROOT_RESPONSE) assert api_root._loaded_information is False assert api_root.title == "Malware Research Group" assert api_root._loaded_information is True assert api_root.description == "A trust group setup for malware researchers" assert api_root.versions == ["taxii-2.0"] assert api_root.max_content_length == 9765625 apiroot_dict = json.loads(API_ROOT_RESPONSE) assert api_root._raw == apiroot_dict @responses.activate def test_api_root_collections(api_root): set_collections_response() assert api_root._loaded_collections is False assert len(api_root.collections) == 2 assert api_root._loaded_collections is True coll = api_root.collections[0] # A collection populated from an API Root is automatically loaded assert coll._loaded is True assert coll.id == "91a7b528-80eb-42ed-a74d-c6fbd5a26116" assert coll.url == COLLECTION_URL assert coll.title == "High Value Indicator Collection" assert coll.description == "This data collection is for collecting high value IOCs" assert coll.can_read is True assert coll.can_write is False assert coll.media_types == [MEDIA_TYPE_STIX_V20] collection_dict = json.loads(COLLECTION_RESPONSE) assert coll._raw == collection_dict @responses.activate def test_collection(collection): assert collection._loaded is False assert collection.id == "91a7b528-80eb-42ed-a74d-c6fbd5a26116" assert collection._loaded is True assert collection.url == COLLECTION_URL assert collection.title == "High Value Indicator Collection" assert collection.description == "This data collection is for collecting high value IOCs" assert collection.can_read is True assert collection.can_write is False assert collection.media_types == [MEDIA_TYPE_STIX_V20] collection_dict = json.loads(COLLECTION_RESPONSE) assert collection._raw == collection_dict def test_collection_unexpected_kwarg(): with pytest.raises(TypeError): Collection(url="", conn=None, foo="bar") @responses.activate def test_get_collection_objects_paged_1(collection): obj_return = [] for x in range(0, 50): obj_return.append(json.loads(STIX_OBJECT)) responses.add(responses.GET, GET_OBJECTS_URL, json.dumps({"type": "bundle", "spec_version": "2.0", "id": "bundle--5d0092c5-5f74-4287-9642-33f4c354e56d", "objects": obj_return[:10]}), status=200, content_type=MEDIA_TYPE_STIX_V20, headers={'Content-Range': 'items 0-9/50'}) responses.add(responses.GET, GET_OBJECTS_URL, json.dumps({"type": "bundle", "spec_version": "2.0", "id": "bundle--5d0092c5-5f74-4287-9642-33f4c354e56d", "objects": obj_return[10:20]}), status=200, content_type=MEDIA_TYPE_STIX_V20, headers={'Content-Range': 'items 10-19/50'}) responses.add(responses.GET, GET_OBJECTS_URL, json.dumps({"type": "bundle", "spec_version": "2.0", "id": "bundle--5d0092c5-5f74-4287-9642-33f4c354e56d", "objects": obj_return[20:30]}), status=200, content_type=MEDIA_TYPE_STIX_V20, headers={'Content-Range': 'items 20-29/50'}) responses.add(responses.GET, GET_OBJECTS_URL, json.dumps({"type": "bundle", "spec_version": "2.0", "id": "bundle--5d0092c5-5f74-4287-9642-33f4c354e56d", "objects": obj_return[30:40]}), status=200, content_type=MEDIA_TYPE_STIX_V20, headers={'Content-Range': 'items 30-39/50'}) responses.add(responses.GET, GET_OBJECTS_URL, json.dumps({"type": "bundle", "spec_version": "2.0", "id": "bundle--5d0092c5-5f74-4287-9642-33f4c354e56d", "objects": obj_return[40:50]}), status=200, content_type=MEDIA_TYPE_STIX_V20, headers={'Content-Range': 'items 40-49/50'}) response = [] for bundle in as_pages(collection.get_objects, per_request=10): response.extend(bundle.get("objects", [])) assert len(response) == 50 @responses.activate def test_get_collection_objects_paged_2(collection): obj_return = [] for x in range(0, 50): obj_return.append(json.loads(STIX_OBJECT)) responses.add(responses.GET, GET_OBJECTS_URL, json.dumps({"type": "bundle", "spec_version": "2.0", "id": "bundle--5d0092c5-5f74-4287-9642-33f4c354e56d", "objects": obj_return[:10]}), status=200, content_type=MEDIA_TYPE_STIX_V20, headers={'Content-Range': 'items 0-9/*'}) responses.add(responses.GET, GET_OBJECTS_URL, json.dumps({"type": "bundle", "spec_version": "2.0", "id": "bundle--5d0092c5-5f74-4287-9642-33f4c354e56d", "objects": obj_return[10:20]}), status=200, content_type=MEDIA_TYPE_STIX_V20, headers={'Content-Range': 'items 10-19/*'}) responses.add(responses.GET, GET_OBJECTS_URL, json.dumps({"type": "bundle", "spec_version": "2.0", "id": "bundle--5d0092c5-5f74-4287-9642-33f4c354e56d", "objects": obj_return[20:30]}), status=200, content_type=MEDIA_TYPE_STIX_V20, headers={'Content-Range': 'items 20-29/*'}) responses.add(responses.GET, GET_OBJECTS_URL, json.dumps({"type": "bundle", "spec_version": "2.0", "id": "bundle--5d0092c5-5f74-4287-9642-33f4c354e56d", "objects": obj_return[30:40]}), status=200, content_type=MEDIA_TYPE_STIX_V20, headers={'Content-Range': 'items 30-39/*'}) responses.add(responses.GET, GET_OBJECTS_URL, json.dumps({"type": "bundle", "spec_version": "2.0", "id": "bundle--5d0092c5-5f74-4287-9642-33f4c354e56d", "objects": obj_return[40:50]}), status=200, content_type=MEDIA_TYPE_STIX_V20, headers={'Content-Range': 'items 40-49/*'}) responses.add(responses.GET, GET_OBJECTS_URL, "", status=406, content_type=MEDIA_TYPE_STIX_V20) response = [] # The status errors (any 400-500) will make it stop with pytest.raises(requests.exceptions.HTTPError): for bundle in as_pages(collection.get_objects, per_request=10): response.extend(bundle.get("objects", [])) assert len(response) == 50 @responses.activate def test_get_collection_objects_paged_3(collection): obj_return = [] for x in range(0, 50): obj_return.append(json.loads(STIX_OBJECT)) responses.add(responses.GET, GET_OBJECTS_URL, json.dumps({"type": "bundle", "spec_version": "2.0", "id": "bundle--5d0092c5-5f74-4287-9642-33f4c354e56d", "objects": obj_return[:10]}), status=200, content_type=MEDIA_TYPE_STIX_V20, headers={'Content-Range': 'items */50'}) responses.add(responses.GET, GET_OBJECTS_URL, json.dumps({"type": "bundle", "spec_version": "2.0", "id": "bundle--5d0092c5-5f74-4287-9642-33f4c354e56d", "objects": obj_return[10:20]}), status=200, content_type=MEDIA_TYPE_STIX_V20, headers={'Content-Range': 'items */50'}) responses.add(responses.GET, GET_OBJECTS_URL, json.dumps({"type": "bundle", "spec_version": "2.0", "id": "bundle--5d0092c5-5f74-4287-9642-33f4c354e56d", "objects": obj_return[20:30]}), status=200, content_type=MEDIA_TYPE_STIX_V20, headers={'Content-Range': 'items */50'}) responses.add(responses.GET, GET_OBJECTS_URL, json.dumps({"type": "bundle", "spec_version": "2.0", "id": "bundle--5d0092c5-5f74-4287-9642-33f4c354e56d", "objects": obj_return[30:40]}), status=200, content_type=MEDIA_TYPE_STIX_V20, headers={'Content-Range': 'items */50'}) responses.add(responses.GET, GET_OBJECTS_URL, json.dumps({"type": "bundle", "spec_version": "2.0", "id": "bundle--5d0092c5-5f74-4287-9642-33f4c354e56d", "objects": obj_return[40:50]}), status=200, content_type=MEDIA_TYPE_STIX_V20, headers={'Content-Range': 'items */50'}) response = [] for bundle in as_pages(collection.get_objects, per_request=10): response.extend(bundle.get("objects", [])) assert len(response) == 50 @responses.activate def test_get_collection_objects_paged_4(collection): obj_return = [] for x in range(0, 50): obj_return.append(json.loads(STIX_OBJECT)) responses.add(responses.GET, GET_OBJECTS_URL, json.dumps({"type": "bundle", "spec_version": "2.0", "id": "bundle--5d0092c5-5f74-4287-9642-33f4c354e56d", "objects": obj_return[:10]}), status=200, content_type=MEDIA_TYPE_STIX_V20, headers={'Content-Range': 'items */*'}) responses.add(responses.GET, GET_OBJECTS_URL, json.dumps({"type": "bundle", "spec_version": "2.0", "id": "bundle--5d0092c5-5f74-4287-9642-33f4c354e56d", "objects": obj_return[10:20]}), status=200, content_type=MEDIA_TYPE_STIX_V20, headers={'Content-Range': 'items */*'}) responses.add(responses.GET, GET_OBJECTS_URL, json.dumps({"type": "bundle", "spec_version": "2.0", "id": "bundle--5d0092c5-5f74-4287-9642-33f4c354e56d", "objects": obj_return[20:30]}), status=200, content_type=MEDIA_TYPE_STIX_V20, headers={'Content-Range': 'items */*'}) responses.add(responses.GET, GET_OBJECTS_URL, json.dumps({"type": "bundle", "spec_version": "2.0", "id": "bundle--5d0092c5-5f74-4287-9642-33f4c354e56d", "objects": obj_return[30:40]}), status=200, content_type=MEDIA_TYPE_STIX_V20, headers={'Content-Range': 'items */*'}) responses.add(responses.GET, GET_OBJECTS_URL, json.dumps({"type": "bundle", "spec_version": "2.0", "id": "bundle--5d0092c5-5f74-4287-9642-33f4c354e56d", "objects": obj_return[40:50]}), status=200, content_type=MEDIA_TYPE_STIX_V20, headers={'Content-Range': 'items */*'}) responses.add(responses.GET, GET_OBJECTS_URL, "", status=406, content_type=MEDIA_TYPE_STIX_V20) response = [] # The status errors (any 400-500) will make it stop with pytest.raises(requests.exceptions.HTTPError): for bundle in as_pages(collection.get_objects, per_request=10): response.extend(bundle.get("objects", [])) assert len(response) == 50 @responses.activate def test_get_collection_objects_paged_5(collection): obj_return = [] for x in range(0, 50): obj_return.append(json.loads(STIX_OBJECT)) responses.add(responses.GET, GET_OBJECTS_URL, json.dumps({"type": "bundle", "spec_version": "2.0", "id": "bundle--5d0092c5-5f74-4287-9642-33f4c354e56d", "objects": obj_return[:10]}), status=200, content_type=MEDIA_TYPE_STIX_V20) responses.add(responses.GET, GET_OBJECTS_URL, json.dumps({"type": "bundle", "spec_version": "2.0", "id": "bundle--5d0092c5-5f74-4287-9642-33f4c354e56d", "objects": obj_return[10:20]}), status=200, content_type=MEDIA_TYPE_STIX_V20) responses.add(responses.GET, GET_OBJECTS_URL, json.dumps({"type": "bundle", "spec_version": "2.0", "id": "bundle--5d0092c5-5f74-4287-9642-33f4c354e56d", "objects": obj_return[20:30]}), status=200, content_type=MEDIA_TYPE_STIX_V20) responses.add(responses.GET, GET_OBJECTS_URL, json.dumps({"type": "bundle", "spec_version": "2.0", "id": "bundle--5d0092c5-5f74-4287-9642-33f4c354e56d", "objects": obj_return[30:40]}), status=200, content_type=MEDIA_TYPE_STIX_V20) responses.add(responses.GET, GET_OBJECTS_URL, json.dumps({"type": "bundle", "spec_version": "2.0", "id": "bundle--5d0092c5-5f74-4287-9642-33f4c354e56d", "objects": obj_return[40:50]}), status=200, content_type=MEDIA_TYPE_STIX_V20) responses.add(responses.GET, GET_OBJECTS_URL, "", status=406, content_type=MEDIA_TYPE_STIX_V20) response = [] # The status errors (any 400-500) will make it stop with pytest.raises(requests.exceptions.HTTPError): for bundle in as_pages(collection.get_objects, per_request=10): response.extend(bundle.get("objects", [])) assert len(response) == 50 @responses.activate def test_get_collection_objects(collection): responses.add(responses.GET, GET_OBJECTS_URL, GET_OBJECTS_RESPONSE, status=200, content_type=MEDIA_TYPE_STIX_V20) response = collection.get_objects() assert response["spec_version"] == "2.0" assert len(response["objects"]) == 1 @responses.activate def test_get_object(collection): responses.add(responses.GET, GET_OBJECT_URL, GET_OBJECT_RESPONSE, status=200, content_type=MEDIA_TYPE_STIX_V20) response = collection.get_object("indicator--252c7c11-daf2-42bd-843b-be65edca9f61") indicator = response["objects"][0] assert indicator["id"] == "indicator--252c7c11-daf2-42bd-843b-be65edca9f61" @responses.activate def test_cannot_write_to_readonly_collection(collection): with pytest.raises(AccessError): collection.add_objects(STIX_BUNDLE) @responses.activate def test_add_object_to_collection(writable_collection): responses.add(responses.POST, ADD_WRITABLE_OBJECTS_URL, ADD_OBJECTS_RESPONSE, status=202, content_type=MEDIA_TYPE_TAXII_V20) status = writable_collection.add_objects(STIX_BUNDLE) assert status.status == "complete" assert status.total_count == 1 assert status.success_count == 1 assert len(status.successes) == 1 assert status.failure_count == 0 assert status.pending_count == 0 status_dict = json.loads(ADD_OBJECTS_RESPONSE) assert status._raw == status_dict @responses.activate def test_add_object_to_collection_dict(writable_collection): responses.add(responses.POST, ADD_WRITABLE_OBJECTS_URL, ADD_OBJECTS_RESPONSE, status=202, content_type=MEDIA_TYPE_TAXII_V20) dict_bundle = json.load(six.StringIO(STIX_BUNDLE)) status = writable_collection.add_objects(dict_bundle) assert status.status == "complete" assert status.total_count == 1 assert status.success_count == 1 assert len(status.successes) == 1 assert status.failure_count == 0 assert status.pending_count == 0 @responses.activate def test_add_object_to_collection_bin(writable_collection): responses.add(responses.POST, ADD_WRITABLE_OBJECTS_URL, ADD_OBJECTS_RESPONSE, status=202, content_type=MEDIA_TYPE_TAXII_V20) bin_bundle = STIX_BUNDLE.encode("utf-8") status = writable_collection.add_objects(bin_bundle) assert status.status == "complete" assert status.total_count == 1 assert status.success_count == 1 assert len(status.successes) == 1 assert status.failure_count == 0 assert status.pending_count == 0 @responses.activate def test_add_object_to_collection_badtype(writable_collection): responses.add(responses.POST, ADD_WRITABLE_OBJECTS_URL, ADD_OBJECTS_RESPONSE, status=202, content_type=MEDIA_TYPE_TAXII_V20) with pytest.raises(TypeError): writable_collection.add_objects([1, 2, 3]) @responses.activate def test_add_object_rases_error_when_collection_id_does_not_match_url( bad_writable_collection): responses.add(responses.POST, ADD_OBJECTS_URL, ADD_OBJECTS_RESPONSE, status=202, content_type=MEDIA_TYPE_TAXII_V20) with pytest.raises(ValidationError) as excinfo: bad_writable_collection.add_objects(STIX_BUNDLE) msg = ("The collection 'e278b87e-0f9b-4c63-a34c-c8f0b3e91acb' does not " "match the url for queries " "'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/'") assert str(excinfo.value) == msg @responses.activate def test_cannot_read_from_writeonly_collection(writable_collection): with pytest.raises(AccessError): writable_collection.get_objects() @responses.activate def test_get_manifest(collection): responses.add(responses.GET, MANIFEST_URL, GET_MANIFEST_RESPONSE, status=200, content_type=MEDIA_TYPE_TAXII_V20) response = collection.get_manifest() assert len(response["objects"]) == 2 obj = response["objects"][0] assert obj["id"] == "indicator--29aba82c-5393-42a8-9edb-6a2cb1df070b" assert len(obj["versions"]) == 2 assert obj["media_types"][0] == MEDIA_TYPE_STIX_V20 @responses.activate def test_get_status(api_root, status_dict): set_status_response() status = api_root.get_status(STATUS_ID) assert status.total_count == 4 assert status.success_count == 1 assert len(status.successes) == 1 assert status.failure_count == 1 assert len(status.failures) == 1 assert status.pending_count == 2 assert len(status.pendings) == 2 assert status._raw == status_dict @responses.activate def test_status_raw(status_dict): """Test Status object created directly (not obtained via ApiRoot), and _raw property.""" set_status_response() status = Status(STATUS_URL) assert status_dict == status._raw @responses.activate def test_content_type_valid(collection): responses.add(responses.GET, GET_OBJECT_URL, GET_OBJECT_RESPONSE, status=200, content_type="%s; charset=utf-8" % MEDIA_TYPE_STIX_V20) response = collection.get_object("indicator--252c7c11-daf2-42bd-843b-be65edca9f61") indicator = response["objects"][0] assert indicator["id"] == "indicator--252c7c11-daf2-42bd-843b-be65edca9f61" @responses.activate def test_content_type_invalid(collection): responses.add(responses.GET, GET_OBJECT_URL, GET_OBJECT_RESPONSE, status=200, content_type="taxii") with pytest.raises(TAXIIServiceException) as excinfo: collection.get_object("indicator--252c7c11-daf2-42bd-843b-be65edca9f61") assert ("Unexpected Response. Got Content-Type: 'taxii' for " "Accept: 'application/vnd.oasis.stix+json; version=2.0'") in str(excinfo.value) def test_url_filter_type(): params = _filter_kwargs_to_query_params({"type": "foo"}) assert params == {"match[type]": "foo"} params = _filter_kwargs_to_query_params({"type": ("foo", "bar")}) assert params == {"match[type]": "foo,bar"} def test_filter_id(): params = _filter_kwargs_to_query_params({"id": "foo"}) assert params == {"match[id]": "foo"} params = _filter_kwargs_to_query_params({"id": ("foo", "bar")}) assert params == {"match[id]": "foo,bar"} def test_filter_version(): params = _filter_kwargs_to_query_params({"version": "foo"}) assert params == {"match[version]": "foo"} dt = datetime.datetime(2010, 9, 8, 7, 6, 5) params = _filter_kwargs_to_query_params({"version": dt}) assert params == {"match[version]": "2010-09-08T07:06:05Z"} params = _filter_kwargs_to_query_params({"version": (dt, "bar")}) assert params == {"match[version]": "2010-09-08T07:06:05Z,bar"} def test_filter_added_after(): params = _filter_kwargs_to_query_params({"added_after": "foo"}) assert params == {"added_after": "foo"} dt = datetime.datetime(2010, 9, 8, 7, 6, 5) params = _filter_kwargs_to_query_params({"added_after": dt}) assert params == {"added_after": "2010-09-08T07:06:05Z"} with pytest.raises(InvalidArgumentsError): _filter_kwargs_to_query_params({"added_after": (dt, "bar")}) def test_filter_combo(): dt = datetime.datetime(2010, 9, 8, 7, 6, 5) params = _filter_kwargs_to_query_params({ "added_after": dt, "type": ("indicator", "malware"), "version": dt, "foo": ("bar", "baz") }) assert params == { "added_after": "2010-09-08T07:06:05Z", "match[type]": "indicator,malware", "match[version]": "2010-09-08T07:06:05Z", "match[foo]": "bar,baz" } def test_params_filter_unknown(): params = _filter_kwargs_to_query_params({"foo": "bar"}) assert params == {"match[foo]": "bar"} def test_taxii_endpoint_raises_exception(): """Test exception is raised when conn and (user or pass) is provided""" conn = _HTTPConnection(user="foo", password="bar", verify=False) error_str = "Only one of a connection, username/password, or auth object may be provided." fake_url = "https://example.com/api1/collections/" with pytest.raises(InvalidArgumentsError) as excinfo: _TAXIIEndpoint(fake_url, conn, "other", "test") assert error_str in str(excinfo.value) with pytest.raises(InvalidArgumentsError) as excinfo: _TAXIIEndpoint(fake_url, conn, auth=TokenAuth('abcd')) assert error_str in str(excinfo.value) with pytest.raises(InvalidArgumentsError) as excinfo: _TAXIIEndpoint(fake_url, user="other", password="test", auth=TokenAuth('abcd')) assert error_str in str(excinfo.value) with pytest.raises(InvalidArgumentsError) as excinfo: _TAXIIEndpoint(fake_url, conn, "other", "test", auth=TokenAuth('abcd')) assert error_str in str(excinfo.value) @responses.activate def test_valid_content_type_for_connection(): """The server responded with charset=utf-8, but the media types are correct and first.""" responses.add(responses.GET, COLLECTION_URL, COLLECTIONS_RESPONSE, status=200, content_type=MEDIA_TYPE_TAXII_V20 + "; charset=utf-8") conn = _HTTPConnection(user="foo", password="bar", verify=False) conn.get("https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/", headers={"Accept": MEDIA_TYPE_TAXII_V20}) @responses.activate def test_invalid_content_type_for_connection(): responses.add(responses.GET, COLLECTION_URL, COLLECTIONS_RESPONSE, status=200, content_type=MEDIA_TYPE_TAXII_V20) with pytest.raises(TAXIIServiceException) as excinfo: conn = _HTTPConnection(user="foo", password="bar", verify=False) conn.get("https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/", headers={"Accept": MEDIA_TYPE_TAXII_V20 + "; charset=utf-8"}) assert ("Unexpected Response. Got Content-Type: 'application/vnd.oasis.taxii+json; " "version=2.0' for Accept: 'application/vnd.oasis.taxii+json; version=2.0; " "charset=utf-8'") in str(excinfo.value) @responses.activate def test_invalid_accept_for_connection(): responses.add(responses.GET, COLLECTION_URL, COLLECTIONS_RESPONSE, status=406, content_type=MEDIA_TYPE_TAXII_V20) with pytest.raises(requests.exceptions.HTTPError): conn = _HTTPConnection(user="foo", password="bar", verify=False) conn.get("https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/", headers={"Accept": "application/taxii+json; version=2.1"}) def test_status_missing_id_property(status_dict): with pytest.raises(ValidationError) as excinfo: status_dict.pop("id") Status("https://example.com/api1/status/12345678-1234-1234-1234-123456789012/", user="foo", password="bar", verify=False, status_info=status_dict) assert "No 'id' in Status for request 'https://example.com/api1/status/12345678-1234-1234-1234-123456789012/'" == str(excinfo.value) def test_status_missing_status_property(status_dict): with pytest.raises(ValidationError) as excinfo: status_dict.pop("status") Status("https://example.com/api1/status/12345678-1234-1234-1234-123456789012/", user="foo", password="bar", verify=False, status_info=status_dict) assert "No 'status' in Status for request 'https://example.com/api1/status/12345678-1234-1234-1234-123456789012/'" == str(excinfo.value) def test_status_missing_total_count_property(status_dict): with pytest.raises(ValidationError) as excinfo: status_dict.pop("total_count") Status("https://example.com/api1/status/12345678-1234-1234-1234-123456789012/", user="foo", password="bar", verify=False, status_info=status_dict) assert "No 'total_count' in Status for request 'https://example.com/api1/status/12345678-1234-1234-1234-123456789012/'" == str(excinfo.value) def test_status_missing_success_count_property(status_dict): with pytest.raises(ValidationError) as excinfo: status_dict.pop("success_count") Status("https://example.com/api1/status/12345678-1234-1234-1234-123456789012/", user="foo", password="bar", verify=False, status_info=status_dict) assert "No 'success_count' in Status for request 'https://example.com/api1/status/12345678-1234-1234-1234-123456789012/'" == str(excinfo.value) def test_status_missing_failure_count_property(status_dict): with pytest.raises(ValidationError) as excinfo: status_dict.pop("failure_count") Status("https://example.com/api1/status/12345678-1234-1234-1234-123456789012/", user="foo", password="bar", verify=False, status_info=status_dict) assert "No 'failure_count' in Status for request 'https://example.com/api1/status/12345678-1234-1234-1234-123456789012/'" == str(excinfo.value) def test_status_missing_pending_count_property(status_dict): with pytest.raises(ValidationError) as excinfo: status_dict.pop("pending_count") Status("https://example.com/api1/status/12345678-1234-1234-1234-123456789012/", user="foo", password="bar", verify=False, status_info=status_dict) assert "No 'pending_count' in Status for request 'https://example.com/api1/status/12345678-1234-1234-1234-123456789012/'" == str(excinfo.value) def test_collection_missing_id_property(collection_dict): with pytest.raises(ValidationError) as excinfo: collection_dict.pop("id") Collection("https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/", user="foo", password="bar", verify=False, collection_info=collection_dict) assert "No 'id' in Collection for request 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/'" == str(excinfo.value) def test_collection_missing_title_property(collection_dict): with pytest.raises(ValidationError) as excinfo: collection_dict.pop("title") Collection("https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/", user="foo", password="bar", verify=False, collection_info=collection_dict) assert "No 'title' in Collection for request 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/'" == str(excinfo.value) def test_collection_missing_can_read_property(collection_dict): with pytest.raises(ValidationError) as excinfo: collection_dict.pop("can_read") Collection("https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/", user="foo", password="bar", verify=False, collection_info=collection_dict) assert "No 'can_read' in Collection for request 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/'" == str(excinfo.value) def test_collection_missing_can_write_property(collection_dict): with pytest.raises(ValidationError) as excinfo: collection_dict.pop("can_write") Collection("https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/", user="foo", password="bar", verify=False, collection_info=collection_dict) assert "No 'can_write' in Collection for request 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/'" == str(excinfo.value) def test_conn_post_kwarg_errors(): conn = _HTTPConnection() with pytest.raises(InvalidArgumentsError): conn.post(DISCOVERY_URL, data=1, json=2) with pytest.raises(InvalidArgumentsError): conn.post(DISCOVERY_URL, data=1, foo=2) with pytest.raises(InvalidArgumentsError): conn.post(DISCOVERY_URL, foo=1) def test_user_agent_defaulting(): conn = _HTTPConnection(user_agent="foo/1.0") headers = conn._merge_headers({}) # also test key access is case-insensitive assert headers["user-agent"] == "foo/1.0" def test_user_agent_overriding(): conn = _HTTPConnection(user_agent="foo/1.0") headers = conn._merge_headers({"User-Agent": "bar/2.0"}) assert headers["user-agent"] == "bar/2.0" def test_user_agent_enforcing1(): conn = _HTTPConnection(user_agent=None) headers = conn._merge_headers({}) assert headers["user-agent"] == DEFAULT_USER_AGENT def test_user_agent_enforcing2(): conn = _HTTPConnection() headers = conn._merge_headers({"User-Agent": None}) assert headers["user-agent"] == DEFAULT_USER_AGENT def test_user_agent_enforcing3(): conn = _HTTPConnection(user_agent=None) headers = conn._merge_headers({"User-Agent": None}) assert headers["user-agent"] == DEFAULT_USER_AGENT def test_header_merging(): conn = _HTTPConnection() headers = conn._merge_headers({"AddedHeader": "addedvalue"}) assert headers == { "user-agent": DEFAULT_USER_AGENT, "addedheader": "addedvalue" } def test_header_merge_none(): conn = _HTTPConnection() headers = conn._merge_headers(None) assert headers == { "user-agent": DEFAULT_USER_AGENT } def test_collection_with_custom_properties(collection_dict): collection_dict["type"] = "domain" col_obj = Collection(url=WRITABLE_COLLECTION_URL, collection_info=collection_dict) assert len(col_obj.custom_properties) == 1 assert col_obj.custom_properties["type"] == "domain" def test_status_with_custom_properties(status_dict): status_dict["x_example_com"] = "some value" status_obj = Status(url=COLLECTION_URL, status_info=status_dict) assert len(status_obj.custom_properties) == 1 assert status_obj.custom_properties["x_example_com"] == "some value" @responses.activate def test_api_roots_with_custom_properties(api_root): response = """{ "title": "Malware Research Group", "description": "A trust group setup for malware researchers", "versions": ["taxii-2.0"], "max_content_length": 9765625, "x_example_com_total_items": 1000 }""" set_api_root_response(response) api_root.refresh_information() assert len(api_root.custom_properties) == 1 assert api_root.custom_properties["x_example_com_total_items"] == 1000 @responses.activate def test_server_with_custom_properties(server): response = """{ "title": "Some TAXII Server", "description": "This TAXII Server contains a listing of...", "contact": "string containing contact information", "default": "https://example.com/api2/", "api_roots": [ "https://example.com/api1/", "https://example.com/api2/", "https://example.net/trustgroup1/" ], "x_example_com": "some value" }""" set_discovery_response(response) server.refresh() assert len(server.custom_properties) == 1 assert server.custom_properties["x_example_com"] == "some value" @responses.activate def test_collection_missing_trailing_slash(): set_collection_response() collection = Collection(COLLECTION_URL[:-1]) responses.add(responses.GET, GET_OBJECT_URL, GET_OBJECT_RESPONSE, status=200, content_type="%s; charset=utf-8" % MEDIA_TYPE_STIX_V20) response = collection.get_object("indicator--252c7c11-daf2-42bd-843b-be65edca9f61") indicator = response["objects"][0] assert indicator["id"] == "indicator--252c7c11-daf2-42bd-843b-be65edca9f61" @responses.activate def test_get_objects_pagination_success(collection): responses.add(responses.GET, GET_OBJECTS_URL, GET_OBJECTS_RESPONSE, status=200, content_type=MEDIA_TYPE_STIX_V20) response = collection.get_objects(per_request=50).json() indicator = response["objects"][0] assert len(response["objects"]) == 1 assert indicator["id"] == "indicator--252c7c11-daf2-42bd-843b-be65edca9f61" @responses.activate def test_get_objects_pagination_fail(collection): error = ERROR_MESSAGE % "400" responses.add(responses.GET, GET_OBJECTS_URL, error, status=400, content_type=MEDIA_TYPE_STIX_V20) with pytest.raises(requests.exceptions.HTTPError) as e: collection.get_objects(per_request=50).json() assert e.value.response.status_code == 400 @responses.activate def test_get_objects_pagination_fail_no_page(collection): error = ERROR_MESSAGE % "400" responses.add(responses.GET, GET_OBJECTS_URL, error, status=400, content_type=MEDIA_TYPE_STIX_V20) with pytest.raises(requests.exceptions.HTTPError) as e: collection.get_objects().json() assert e.value.response.status_code == 400 @responses.activate def test_get_manifests_pagination_success(collection): responses.add(responses.GET, MANIFEST_URL, GET_MANIFEST_RESPONSE, status=200, content_type=MEDIA_TYPE_TAXII_V20) response = collection.get_manifest(per_request=50).json() assert len(response["objects"]) == 2 obj = response["objects"][0] assert obj["id"] == "indicator--29aba82c-5393-42a8-9edb-6a2cb1df070b" assert len(obj["versions"]) == 2 assert obj["media_types"][0] == MEDIA_TYPE_STIX_V20 @responses.activate def test_get_manifests_pagination_fail(collection): error = ERROR_MESSAGE % "400" responses.add(responses.GET, MANIFEST_URL, error, status=400, content_type=MEDIA_TYPE_TAXII_V20) with pytest.raises(requests.exceptions.HTTPError) as e: collection.get_manifest(per_request=50).json() assert e.value.response.status_code == 400 @responses.activate def test_get_manifests_pagination_fail_no_page(collection): error = ERROR_MESSAGE % "400" responses.add(responses.GET, MANIFEST_URL, error, status=400, content_type=MEDIA_TYPE_TAXII_V20) with pytest.raises(requests.exceptions.HTTPError) as e: collection.get_manifest().json() assert e.value.response.status_code == 400 cti-taxii-client-2.3.0/taxii2client/test/test_client_v21.py000066400000000000000000001075631402275443100236360ustar00rootroot00000000000000import datetime import json import pytest import requests import responses import six from taxii2client import DEFAULT_USER_AGENT, MEDIA_TYPE_TAXII_V21 from taxii2client.common import ( TokenAuth, _filter_kwargs_to_query_params, _HTTPConnection, _TAXIIEndpoint ) from taxii2client.exceptions import ( AccessError, InvalidArgumentsError, InvalidJSONError, TAXIIServiceException, ValidationError ) from taxii2client.v21 import ApiRoot, Collection, Server, Status, as_pages MEDIA_TYPE_STIX_V21 = "application/stix+json;version=2.1" TAXII_SERVER = "example.com" DISCOVERY_URL = "https://{}/taxii2/".format(TAXII_SERVER) API_ROOT_URL = "https://{}/api1/".format(TAXII_SERVER) COLLECTIONS_URL = API_ROOT_URL + "collections/" COLLECTION_URL = COLLECTIONS_URL + "91a7b528-80eb-42ed-a74d-c6fbd5a26116/" OBJECTS_URL = COLLECTION_URL + "objects/" GET_OBJECTS_URL = OBJECTS_URL ADD_OBJECTS_URL = OBJECTS_URL WRITABLE_COLLECTION_URL = COLLECTIONS_URL + "e278b87e-0f9b-4c63-a34c-c8f0b3e91acb/" ADD_WRITABLE_OBJECTS_URL = WRITABLE_COLLECTION_URL + "objects/" GET_OBJECT_URL = OBJECTS_URL + "indicator--252c7c11-daf2-42bd-843b-be65edca9f61/" MANIFEST_URL = COLLECTION_URL + "manifest/" STATUS_ID = "2d086da7-4bdc-4f91-900e-d77486753710" STATUS_URL = API_ROOT_URL + "status/" + STATUS_ID + "/" # These responses are provided as examples in the TAXII 2.0 specification. DISCOVERY_RESPONSE = """{ "title": "Some TAXII Server", "description": "This TAXII Server contains a listing of...", "contact": "string containing contact information", "default": "https://example.com/api2/", "api_roots": [ "https://example.com/api1/", "https://example.com/api2/", "https://example.net/trustgroup1/" ] }""" API_ROOT_RESPONSE = """{ "title": "Malware Research Group", "description": "A trust group setup for malware researchers", "versions": ["application/taxii+json;version=2.1"], "max_content_length": 9765625 }""" COLLECTIONS_RESPONSE = """{ "collections": [ { "id": "91a7b528-80eb-42ed-a74d-c6fbd5a26116", "title": "High Value Indicator Collection", "description": "This data collection is for collecting high value IOCs", "can_read": true, "can_write": false, "media_types": [ "application/stix+json;version=2.1" ] }, { "id": "52892447-4d7e-4f70-b94d-d7f22742ff63", "title": "Indicators from the past 24-hours", "description": "This data collection is for collecting current IOCs", "can_read": true, "can_write": false, "media_types": [ "application/stix+json;version=2.1" ] } ] }""" COLLECTION_RESPONSE = """{ "id": "91a7b528-80eb-42ed-a74d-c6fbd5a26116", "title": "High Value Indicator Collection", "description": "This data collection is for collecting high value IOCs", "can_read": true, "can_write": false, "media_types": [ "application/stix+json;version=2.1" ] }""" # This collection is not in the spec. WRITABLE_COLLECTION = """{ "id": "e278b87e-0f9b-4c63-a34c-c8f0b3e91acb", "title": "Writable Collection", "description": "This collection is a dropbox for submitting indicators", "can_read": false, "can_write": true, "media_types": [ "application/stix+json;version=2.1" ] }""" STIX_OBJECT = """ { "type": "indicator", "id": "indicator--252c7c11-daf2-42bd-843b-be65edca9f61", "spec_version": "2.1", "created": "2016-04-06T20:03:48.000Z", "modified": "2016-04-06T20:03:48.000Z", "pattern": "[ file:hashes.MD5 = 'd41d8cd98f00b204e9800998ecf8427e' ]", "pattern_type": "stix", "valid_from": "2016-01-01T00:00:00Z" } """ # This bundle is used as the response to get_objects(), and also the bundle # POST'ed with add_objects(). STIX_ENVELOPE = f"""{{ "objects": [ {STIX_OBJECT} ] }}""" GET_OBJECTS_RESPONSE = STIX_ENVELOPE # get_object() still returns a bundle. In this case, the bundle has only one # object (the correct one.) GET_OBJECT_RESPONSE = GET_OBJECTS_RESPONSE # This is the expected response when calling ADD_OBJECTS with the STIX_BUNDLE # above. There is only one object, and it was added successfully. This response # is not in the spec. ADD_OBJECTS_RESPONSE = """{ "id": "350dae03-d2d8-4bd3-bc1d-8160589693e3", "status": "complete", "request_timestamp": "2016-11-02T12:34:34.12345Z", "total_count": 1, "success_count": 1, "successes": [ { "id": "indicator--252c7c11-daf2-42bd-843b-be65edca9f61" } ], "failure_count": 0, "pending_count": 0 }""" # This is the response in Section 5.4 of the spec. It implies a larger # bundle than what is provided in the example. ADD_OBJECTS_RESPONSE_FROM_SPEC = """{ "id": "2d086da7-4bdc-4f91-900e-d77486753710", "status": "pending", "request_timestamp": "2016-11-02T12:34:34.12345Z", "total_count": 4, "success_count": 1, "successes": [ { "id": "indicator--c410e480-e42b-47d1-9476-85307c12bcbf" } ], "failure_count": 0, "pending_count": 3 }""" GET_MANIFEST_RESPONSE = """{ "objects": [ { "id": "indicator--29aba82c-5393-42a8-9edb-6a2cb1df070b", "date_added": "2016-11-04T03:04:051Z", "version": "2016-11-03T12:30:59.000Z", "media_type": "application/stix+json;version=2.1" }, { "id": "indicator--ef0b28e1-308c-4a30-8770-9b4851b260a5", "date_added": "2016-11-04T10:29:061Z", "version": "2016-11-03T12:35:10.000Z", "media_type": "application/stix+json;version=2.1" } ] }""" STATUS_RESPONSE = """{ "id": "2d086da7-4bdc-4f91-900e-d77486753710", "status": "pending", "request_timestamp": "2016-11-02T12:34:34.12345Z", "total_count": 4, "success_count": 1, "successes": [ { "id": "indicator--c410e480-e42b-47d1-9476-85307c12bcbf" , "version": "2018-05-27T12:02:41.312Z" } ], "failure_count": 1, "failures": [ { "id": "malware--664fa29d-bf65-4f28-a667-bdb76f29ec98", "version": "2018-05-28T14:03:42.543Z", "message": "Unable to process object" } ], "pending_count": 2, "pendings": [ { "id": "indicator--252c7c11-daf2-42bd-843b-be65edca9f61", "version": "2018-05-18T20:16:21.148Z" }, { "id": "relationship--045585ad-a22f-4333-af33-bfd503a683b5", "version": "2018-05-15T10:13:32.579Z" } ] }""" BAD_DISCOVERY_RESPONSE = """{"title":""" @pytest.fixture def status_dict(): return { "id": "2d086da7-4bdc-4f91-900e-d77486753710", "status": "pending", "request_timestamp": "2016-11-02T12:34:34.12345Z", "total_count": 4, "success_count": 1, "successes": [ { "id": "indicator--c410e480-e42b-47d1-9476-85307c12bcbf", "version": "2018-05-27T12:02:41.312Z" } ], "failure_count": 1, "failures": [ { "id": "malware--664fa29d-bf65-4f28-a667-bdb76f29ec98", "version": "2018-05-28T14:03:42.543Z", "message": "Unable to process object" } ], "pending_count": 2, "pendings": [ { "id": "indicator--252c7c11-daf2-42bd-843b-be65edca9f61", "version": "2018-05-18T20:16:21.148Z" }, { "id": "relationship--045585ad-a22f-4333-af33-bfd503a683b5", "version": "2018-05-15T10:13:32.579Z" } ] } @pytest.fixture def collection_dict(): return { "id": "e278b87e-0f9b-4c63-a34c-c8f0b3e91acb", "title": "Writable Collection", "description": "This collection is a dropbox for submitting indicators", "can_read": False, "can_write": True, "media_types": [ MEDIA_TYPE_STIX_V21 ] } @pytest.fixture def server(): """Default server object for example.com""" return Server(DISCOVERY_URL, user="foo", password="bar") @pytest.fixture def api_root(): """Default API Root object""" return ApiRoot(API_ROOT_URL) @pytest.fixture def collection(): """Default Collection object""" # The collection response is needed to get information about the collection set_collection_response() return Collection(COLLECTION_URL) @pytest.fixture def writable_collection(): """Collection with 'can_write' set to 'true'.""" set_collection_response(WRITABLE_COLLECTION_URL, WRITABLE_COLLECTION) return Collection(WRITABLE_COLLECTION_URL) @pytest.fixture def bad_writable_collection(): """Collection with 'can_write=true', but the COLLECTION_URL is different from the one in the response""" set_collection_response(response=WRITABLE_COLLECTION) return Collection(COLLECTION_URL) def set_api_root_response(response): responses.add(responses.GET, API_ROOT_URL, body=response, status=200, content_type=MEDIA_TYPE_TAXII_V21) def set_discovery_response(response): responses.add(responses.GET, DISCOVERY_URL, body=response, status=200, content_type=MEDIA_TYPE_TAXII_V21) def set_collections_response(): responses.add(responses.GET, COLLECTIONS_URL, COLLECTIONS_RESPONSE, status=200, content_type=MEDIA_TYPE_TAXII_V21) def set_collection_response(url=COLLECTION_URL, response=COLLECTION_RESPONSE): responses.add(responses.GET, url, response, status=200, content_type=MEDIA_TYPE_TAXII_V21) def set_status_response(): responses.add(responses.GET, STATUS_URL, STATUS_RESPONSE, status=200, content_type=MEDIA_TYPE_TAXII_V21) @responses.activate def test_server_discovery(server): set_discovery_response(DISCOVERY_RESPONSE) assert server._loaded is False assert server.title == "Some TAXII Server" assert server._loaded is True assert server.description == "This TAXII Server contains a listing of..." assert server.contact == "string containing contact information" assert len(server.api_roots) == 3 assert server.default is not None assert server.api_roots[1] == server.default api_root = server.api_roots[0] assert api_root.url == API_ROOT_URL assert api_root._loaded_information is False assert api_root._loaded_collections is False discovery_dict = json.loads(DISCOVERY_RESPONSE) assert server._raw == discovery_dict @responses.activate def test_bad_json_response(server): set_discovery_response(BAD_DISCOVERY_RESPONSE) with pytest.raises(InvalidJSONError): # Just do something to trigger a request server.title @responses.activate def test_minimal_discovery_response(server): # `title` is the only required field on a Discovery Response set_discovery_response('{"title": "Some TAXII Server"}') assert server.title == "Some TAXII Server" assert server.description is None assert server.contact is None assert server.api_roots == [] assert server.default is None @responses.activate def test_discovery_with_no_default(server): response = """{ "title": "Some TAXII Server", "description": "This TAXII Server contains a listing of...", "contact": "string containing contact information", "api_roots": [ "https://example.com/api1/", "https://example.com/api2/", "https://example.net/trustgroup1/" ] }""" set_discovery_response(response) assert len(server.api_roots) == 3 assert server.default is None @responses.activate def test_discovery_with_no_title(server): response = """{ "description": "This TAXII Server contains a listing of...", "contact": "string containing contact information", "api_roots": [ "https://example.com/api1/", "https://example.com/api2/", "https://example.net/trustgroup1/" ] }""" set_discovery_response(response) with pytest.raises(ValidationError) as excinfo: server.refresh() assert "No 'title' in Server Discovery for request 'https://example.com/taxii2/'" == str(excinfo.value) @responses.activate def test_api_root_no_title(api_root): set_api_root_response("""{ "description": "A trust group setup for malware researchers", "versions": ["application/stix+json;version=2.1"], "max_content_length": 9765625 }""") with pytest.raises(ValidationError) as excinfo: assert api_root._loaded_information is False api_root.refresh_information() assert "No 'title' in API Root for request 'https://example.com/api1/'" == str(excinfo.value) @responses.activate def test_api_root_no_versions(api_root): set_api_root_response("""{ "title": "Malware Research Group", "description": "A trust group setup for malware researchers", "max_content_length": 9765625 }""") with pytest.raises(ValidationError) as excinfo: assert api_root._loaded_information is False api_root.refresh_information() assert "No 'versions' in API Root for request 'https://example.com/api1/'" == str(excinfo.value) @responses.activate def test_api_root_no_max_content_length(api_root): set_api_root_response("""{ "title": "Malware Research Group", "description": "A trust group setup for malware researchers", "versions": ["taxii-2.0"] }""") with pytest.raises(ValidationError) as excinfo: assert api_root._loaded_information is False api_root.refresh_information() assert "No 'max_content_length' in API Root for request 'https://example.com/api1/'" == str(excinfo.value) @responses.activate def test_api_root(api_root): set_api_root_response(API_ROOT_RESPONSE) assert api_root._loaded_information is False assert api_root.title == "Malware Research Group" assert api_root._loaded_information is True assert api_root.description == "A trust group setup for malware researchers" assert api_root.versions == ["application/taxii+json;version=2.1"] assert api_root.max_content_length == 9765625 apiroot_dict = json.loads(API_ROOT_RESPONSE) assert api_root._raw == apiroot_dict @responses.activate def test_api_root_collections(api_root): set_collections_response() assert api_root._loaded_collections is False assert len(api_root.collections) == 2 assert api_root._loaded_collections is True coll = api_root.collections[0] # A collection populated from an API Root is automatically loaded assert coll._loaded is True assert coll.id == "91a7b528-80eb-42ed-a74d-c6fbd5a26116" assert coll.url == COLLECTION_URL assert coll.title == "High Value Indicator Collection" assert coll.description == "This data collection is for collecting high value IOCs" assert coll.can_read is True assert coll.can_write is False assert coll.media_types == [MEDIA_TYPE_STIX_V21] collection_dict = json.loads(COLLECTION_RESPONSE) assert coll._raw == collection_dict @responses.activate def test_collection(collection): assert collection._loaded is False assert collection.id == "91a7b528-80eb-42ed-a74d-c6fbd5a26116" assert collection._loaded is True assert collection.url == COLLECTION_URL assert collection.title == "High Value Indicator Collection" assert collection.description == "This data collection is for collecting high value IOCs" assert collection.can_read is True assert collection.can_write is False assert collection.media_types == [MEDIA_TYPE_STIX_V21] collection_dict = json.loads(COLLECTION_RESPONSE) assert collection._raw == collection_dict def test_collection_unexpected_kwarg(): with pytest.raises(TypeError): Collection(url="", conn=None, foo="bar") @responses.activate def test_get_collection_objects(collection): responses.add(responses.GET, GET_OBJECTS_URL, GET_OBJECTS_RESPONSE, status=200, content_type=MEDIA_TYPE_TAXII_V21) response = collection.get_objects() assert len(response["objects"]) == 1 @responses.activate def test_get_collection_objects_paged_1(collection): responses.add(responses.GET, GET_OBJECTS_URL, GET_OBJECTS_RESPONSE, status=200, content_type=MEDIA_TYPE_TAXII_V21) response = [] for bundle in as_pages(collection.get_objects, per_request=50): response.extend(bundle.get("objects", [])) assert len(response) == 1 @responses.activate def test_get_collection_objects_paged_2(collection): obj_return = [] for x in range(0, 50): obj_return.append(json.loads(STIX_OBJECT)) responses.add(responses.GET, GET_OBJECTS_URL, json.dumps({"more": True, "objects": obj_return[:25]}), status=200, content_type=MEDIA_TYPE_TAXII_V21) responses.add(responses.GET, GET_OBJECTS_URL, json.dumps({"more": False, "objects": obj_return[25:]}), status=200, content_type=MEDIA_TYPE_TAXII_V21) response = [] for bundle in as_pages(collection.get_objects, per_request=25): response.extend(bundle.get("objects", [])) assert len(response) == 50 @responses.activate def test_get_object(collection): responses.add(responses.GET, GET_OBJECT_URL, GET_OBJECT_RESPONSE, status=200, content_type=MEDIA_TYPE_TAXII_V21) response = collection.get_object("indicator--252c7c11-daf2-42bd-843b-be65edca9f61") indicator = response["objects"][0] assert indicator["id"] == "indicator--252c7c11-daf2-42bd-843b-be65edca9f61" @responses.activate def test_cannot_write_to_readonly_collection(collection): with pytest.raises(AccessError): collection.add_objects(STIX_ENVELOPE) @responses.activate def test_add_object_to_collection(writable_collection): responses.add(responses.POST, ADD_WRITABLE_OBJECTS_URL, ADD_OBJECTS_RESPONSE, status=202, content_type=MEDIA_TYPE_TAXII_V21) status = writable_collection.add_objects(STIX_ENVELOPE) assert status.status == "complete" assert status.total_count == 1 assert status.success_count == 1 assert len(status.successes) == 1 assert status.failure_count == 0 assert status.pending_count == 0 status_dict = json.loads(ADD_OBJECTS_RESPONSE) assert status._raw == status_dict @responses.activate def test_add_object_to_collection_dict(writable_collection): responses.add(responses.POST, ADD_WRITABLE_OBJECTS_URL, ADD_OBJECTS_RESPONSE, status=202, content_type=MEDIA_TYPE_TAXII_V21) dict_bundle = json.load(six.StringIO(STIX_ENVELOPE)) status = writable_collection.add_objects(dict_bundle) assert status.status == "complete" assert status.total_count == 1 assert status.success_count == 1 assert len(status.successes) == 1 assert status.failure_count == 0 assert status.pending_count == 0 @responses.activate def test_add_object_to_collection_bin(writable_collection): responses.add(responses.POST, ADD_WRITABLE_OBJECTS_URL, ADD_OBJECTS_RESPONSE, status=202, content_type=MEDIA_TYPE_TAXII_V21) bin_bundle = STIX_ENVELOPE.encode("utf-8") status = writable_collection.add_objects(bin_bundle) assert status.status == "complete" assert status.total_count == 1 assert status.success_count == 1 assert len(status.successes) == 1 assert status.failure_count == 0 assert status.pending_count == 0 @responses.activate def test_add_object_to_collection_badtype(writable_collection): responses.add(responses.POST, ADD_WRITABLE_OBJECTS_URL, ADD_OBJECTS_RESPONSE, status=202, content_type=MEDIA_TYPE_TAXII_V21) with pytest.raises(TypeError): writable_collection.add_objects([1, 2, 3]) @responses.activate def test_add_object_rases_error_when_collection_id_does_not_match_url( bad_writable_collection): responses.add(responses.POST, ADD_OBJECTS_URL, ADD_OBJECTS_RESPONSE, status=202, content_type=MEDIA_TYPE_TAXII_V21) with pytest.raises(ValidationError) as excinfo: bad_writable_collection.add_objects(STIX_ENVELOPE) msg = ("The collection 'e278b87e-0f9b-4c63-a34c-c8f0b3e91acb' does not " "match the url for queries " "'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/'") assert str(excinfo.value) == msg @responses.activate def test_cannot_read_from_writeonly_collection(writable_collection): with pytest.raises(AccessError): writable_collection.get_objects() @responses.activate def test_get_manifest(collection): responses.add(responses.GET, MANIFEST_URL, GET_MANIFEST_RESPONSE, status=200, content_type=MEDIA_TYPE_TAXII_V21) response = collection.get_manifest() assert len(response["objects"]) == 2 obj = response["objects"][0] assert obj["id"] == "indicator--29aba82c-5393-42a8-9edb-6a2cb1df070b" assert obj["media_type"] == MEDIA_TYPE_STIX_V21 @responses.activate def test_get_status(api_root, status_dict): set_status_response() status = api_root.get_status(STATUS_ID) assert status.total_count == 4 assert status.success_count == 1 assert len(status.successes) == 1 assert status.failure_count == 1 assert len(status.failures) == 1 assert status.pending_count == 2 assert len(status.pendings) == 2 assert status._raw == status_dict @responses.activate def test_status_raw(status_dict): """Test Status object created directly (not obtained via ApiRoot), and _raw property.""" set_status_response() status = Status(STATUS_URL) assert status_dict == status._raw @responses.activate def test_content_type_valid(collection): responses.add(responses.GET, GET_OBJECT_URL, GET_OBJECT_RESPONSE, status=200, content_type="%s; charset=utf-8" % MEDIA_TYPE_TAXII_V21) response = collection.get_object("indicator--252c7c11-daf2-42bd-843b-be65edca9f61") indicator = response["objects"][0] assert indicator["id"] == "indicator--252c7c11-daf2-42bd-843b-be65edca9f61" @responses.activate def test_content_type_invalid(collection): responses.add(responses.GET, GET_OBJECT_URL, GET_OBJECT_RESPONSE, status=200, content_type="taxii") with pytest.raises(TAXIIServiceException) as excinfo: collection.get_object("indicator--252c7c11-daf2-42bd-843b-be65edca9f61") assert ("Unexpected Response. Got Content-Type: 'taxii' for " "Accept: 'application/taxii+json;version=2.1'") in str(excinfo.value) def test_url_filter_type(): params = _filter_kwargs_to_query_params({"type": "foo"}) assert params == {"match[type]": "foo"} params = _filter_kwargs_to_query_params({"type": ("foo", "bar")}) assert params == {"match[type]": "foo,bar"} def test_filter_id(): params = _filter_kwargs_to_query_params({"id": "foo"}) assert params == {"match[id]": "foo"} params = _filter_kwargs_to_query_params({"id": ("foo", "bar")}) assert params == {"match[id]": "foo,bar"} def test_filter_version(): params = _filter_kwargs_to_query_params({"version": "foo"}) assert params == {"match[version]": "foo"} dt = datetime.datetime(2010, 9, 8, 7, 6, 5) params = _filter_kwargs_to_query_params({"version": dt}) assert params == {"match[version]": "2010-09-08T07:06:05Z"} params = _filter_kwargs_to_query_params({"version": (dt, "bar")}) assert params == {"match[version]": "2010-09-08T07:06:05Z,bar"} def test_filter_added_after(): params = _filter_kwargs_to_query_params({"added_after": "foo"}) assert params == {"added_after": "foo"} dt = datetime.datetime(2010, 9, 8, 7, 6, 5) params = _filter_kwargs_to_query_params({"added_after": dt}) assert params == {"added_after": "2010-09-08T07:06:05Z"} with pytest.raises(InvalidArgumentsError): _filter_kwargs_to_query_params({"added_after": (dt, "bar")}) def test_filter_combo(): dt = datetime.datetime(2010, 9, 8, 7, 6, 5) params = _filter_kwargs_to_query_params({ "added_after": dt, "type": ("indicator", "malware"), "version": dt, "foo": ("bar", "baz") }) assert params == { "added_after": "2010-09-08T07:06:05Z", "match[type]": "indicator,malware", "match[version]": "2010-09-08T07:06:05Z", "match[foo]": "bar,baz" } def test_params_filter_unknown(): params = _filter_kwargs_to_query_params({"foo": "bar"}) assert params == {"match[foo]": "bar"} def test_taxii_endpoint_raises_exception(): """Test exception is raised when conn and (user or pass) is provided""" conn = _HTTPConnection(user="foo", password="bar", verify=False) error_str = "Only one of a connection, username/password, or auth object may be provided." fake_url = "https://example.com/api1/collections/" with pytest.raises(InvalidArgumentsError) as excinfo: _TAXIIEndpoint(fake_url, conn, "other", "test") assert error_str in str(excinfo.value) with pytest.raises(InvalidArgumentsError) as excinfo: _TAXIIEndpoint(fake_url, conn, auth=TokenAuth('abcd')) assert error_str in str(excinfo.value) with pytest.raises(InvalidArgumentsError) as excinfo: _TAXIIEndpoint(fake_url, user="other", password="test", auth=TokenAuth('abcd')) assert error_str in str(excinfo.value) with pytest.raises(InvalidArgumentsError) as excinfo: _TAXIIEndpoint(fake_url, conn, "other", "test", auth=TokenAuth('abcd')) assert error_str in str(excinfo.value) @responses.activate def test_valid_content_type_for_connection(): """The server responded with charset=utf-8, but the media types are correct and first.""" responses.add(responses.GET, COLLECTION_URL, COLLECTIONS_RESPONSE, status=200, content_type=MEDIA_TYPE_TAXII_V21 + "; charset=utf-8") conn = _HTTPConnection(user="foo", password="bar", verify=False, version="2.1") conn.get("https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/", headers={"Accept": MEDIA_TYPE_TAXII_V21}) @responses.activate def test_invalid_content_type_for_connection(): responses.add(responses.GET, COLLECTION_URL, COLLECTIONS_RESPONSE, status=200, content_type=MEDIA_TYPE_TAXII_V21) with pytest.raises(TAXIIServiceException) as excinfo: conn = _HTTPConnection(user="foo", password="bar", verify=False) conn.get("https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/", headers={"Accept": MEDIA_TYPE_TAXII_V21 + "; charset=utf-8"}) assert ("Unexpected Response. Got Content-Type: 'application/taxii+json;" "version=2.1' for Accept: 'application/taxii+json;version=2.1; " "charset=utf-8'") in str(excinfo.value) @responses.activate def test_invalid_accept_for_connection(): responses.add(responses.GET, COLLECTION_URL, COLLECTIONS_RESPONSE, status=406, content_type=MEDIA_TYPE_TAXII_V21) with pytest.raises(requests.exceptions.HTTPError): conn = _HTTPConnection(user="foo", password="bar", verify=False) conn.get("https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/", headers={"Accept": "application/taxii+json; version=2.1"}) def test_status_missing_id_property(status_dict): with pytest.raises(ValidationError) as excinfo: status_dict.pop("id") Status("https://example.com/api1/status/12345678-1234-1234-1234-123456789012/", user="foo", password="bar", verify=False, status_info=status_dict) assert "No 'id' in Status for request 'https://example.com/api1/status/12345678-1234-1234-1234-123456789012/'" == str(excinfo.value) def test_status_missing_status_property(status_dict): with pytest.raises(ValidationError) as excinfo: status_dict.pop("status") Status("https://example.com/api1/status/12345678-1234-1234-1234-123456789012/", user="foo", password="bar", verify=False, status_info=status_dict) assert "No 'status' in Status for request 'https://example.com/api1/status/12345678-1234-1234-1234-123456789012/'" == str(excinfo.value) def test_status_missing_total_count_property(status_dict): with pytest.raises(ValidationError) as excinfo: status_dict.pop("total_count") Status("https://example.com/api1/status/12345678-1234-1234-1234-123456789012/", user="foo", password="bar", verify=False, status_info=status_dict) assert "No 'total_count' in Status for request 'https://example.com/api1/status/12345678-1234-1234-1234-123456789012/'" == str(excinfo.value) def test_status_missing_success_count_property(status_dict): with pytest.raises(ValidationError) as excinfo: status_dict.pop("success_count") Status("https://example.com/api1/status/12345678-1234-1234-1234-123456789012/", user="foo", password="bar", verify=False, status_info=status_dict) assert "No 'success_count' in Status for request 'https://example.com/api1/status/12345678-1234-1234-1234-123456789012/'" == str(excinfo.value) def test_status_missing_failure_count_property(status_dict): with pytest.raises(ValidationError) as excinfo: status_dict.pop("failure_count") Status("https://example.com/api1/status/12345678-1234-1234-1234-123456789012/", user="foo", password="bar", verify=False, status_info=status_dict) assert "No 'failure_count' in Status for request 'https://example.com/api1/status/12345678-1234-1234-1234-123456789012/'" == str(excinfo.value) def test_status_missing_pending_count_property(status_dict): with pytest.raises(ValidationError) as excinfo: status_dict.pop("pending_count") Status("https://example.com/api1/status/12345678-1234-1234-1234-123456789012/", user="foo", password="bar", verify=False, status_info=status_dict) assert "No 'pending_count' in Status for request 'https://example.com/api1/status/12345678-1234-1234-1234-123456789012/'" == str(excinfo.value) def test_collection_missing_id_property(collection_dict): with pytest.raises(ValidationError) as excinfo: collection_dict.pop("id") Collection("https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/", user="foo", password="bar", verify=False, collection_info=collection_dict) assert "No 'id' in Collection for request 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/'" == str(excinfo.value) def test_collection_missing_title_property(collection_dict): with pytest.raises(ValidationError) as excinfo: collection_dict.pop("title") Collection("https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/", user="foo", password="bar", verify=False, collection_info=collection_dict) assert "No 'title' in Collection for request 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/'" == str(excinfo.value) def test_collection_missing_can_read_property(collection_dict): with pytest.raises(ValidationError) as excinfo: collection_dict.pop("can_read") Collection("https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/", user="foo", password="bar", verify=False, collection_info=collection_dict) assert "No 'can_read' in Collection for request 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/'" == str(excinfo.value) def test_collection_missing_can_write_property(collection_dict): with pytest.raises(ValidationError) as excinfo: collection_dict.pop("can_write") Collection("https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/", user="foo", password="bar", verify=False, collection_info=collection_dict) assert "No 'can_write' in Collection for request 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/'" == str(excinfo.value) def test_conn_post_kwarg_errors(): conn = _HTTPConnection() with pytest.raises(InvalidArgumentsError): conn.post(DISCOVERY_URL, data=1, json=2) with pytest.raises(InvalidArgumentsError): conn.post(DISCOVERY_URL, data=1, foo=2) with pytest.raises(InvalidArgumentsError): conn.post(DISCOVERY_URL, foo=1) def test_user_agent_defaulting(): conn = _HTTPConnection(user_agent="foo/1.0") headers = conn._merge_headers({}) # also test key access is case-insensitive assert headers["user-agent"] == "foo/1.0" def test_user_agent_overriding(): conn = _HTTPConnection(user_agent="foo/1.0") headers = conn._merge_headers({"User-Agent": "bar/2.0"}) assert headers["user-agent"] == "bar/2.0" def test_user_agent_enforcing1(): conn = _HTTPConnection(user_agent=None) headers = conn._merge_headers({}) assert headers["user-agent"] == DEFAULT_USER_AGENT def test_user_agent_enforcing2(): conn = _HTTPConnection() headers = conn._merge_headers({"User-Agent": None}) assert headers["user-agent"] == DEFAULT_USER_AGENT def test_user_agent_enforcing3(): conn = _HTTPConnection(user_agent=None) headers = conn._merge_headers({"User-Agent": None}) assert headers["user-agent"] == DEFAULT_USER_AGENT def test_header_merging(): conn = _HTTPConnection() headers = conn._merge_headers({"AddedHeader": "addedvalue"}) assert headers == { "user-agent": DEFAULT_USER_AGENT, "addedheader": "addedvalue" } def test_header_merge_none(): conn = _HTTPConnection() headers = conn._merge_headers(None) assert headers == { "user-agent": DEFAULT_USER_AGENT } def test_collection_with_custom_properties(collection_dict): collection_dict["type"] = "domain" col_obj = Collection(url=WRITABLE_COLLECTION_URL, collection_info=collection_dict) assert len(col_obj.custom_properties) == 1 assert col_obj.custom_properties["type"] == "domain" def test_status_with_custom_properties(status_dict): status_dict["x_example_com"] = "some value" status_obj = Status(url=COLLECTION_URL, status_info=status_dict) assert len(status_obj.custom_properties) == 1 assert status_obj.custom_properties["x_example_com"] == "some value" @responses.activate def test_api_roots_with_custom_properties(api_root): response = """{ "title": "Malware Research Group", "description": "A trust group setup for malware researchers", "versions": ["taxii-2.0"], "max_content_length": 9765625, "x_example_com_total_items": 1000 }""" set_api_root_response(response) api_root.refresh_information() assert len(api_root.custom_properties) == 1 assert api_root.custom_properties["x_example_com_total_items"] == 1000 @responses.activate def test_server_with_custom_properties(server): response = """{ "title": "Some TAXII Server", "description": "This TAXII Server contains a listing of...", "contact": "string containing contact information", "default": "https://example.com/api2/", "api_roots": [ "https://example.com/api1/", "https://example.com/api2/", "https://example.net/trustgroup1/" ], "x_example_com": "some value" }""" set_discovery_response(response) server.refresh() assert len(server.custom_properties) == 1 assert server.custom_properties["x_example_com"] == "some value" @responses.activate def test_collection_missing_trailing_slash(): set_collection_response() collection = Collection(COLLECTION_URL[:-1]) responses.add(responses.GET, GET_OBJECT_URL, GET_OBJECT_RESPONSE, status=200, content_type="%s; charset=utf-8" % MEDIA_TYPE_TAXII_V21) response = collection.get_object("indicator--252c7c11-daf2-42bd-843b-be65edca9f61") indicator = response["objects"][0] assert indicator["id"] == "indicator--252c7c11-daf2-42bd-843b-be65edca9f61" cti-taxii-client-2.3.0/taxii2client/v20/000077500000000000000000000000001402275443100176735ustar00rootroot00000000000000cti-taxii-client-2.3.0/taxii2client/v20/__init__.py000066400000000000000000000760671402275443100220240ustar00rootroot00000000000000"""Python TAXII 2.0 Client API""" from __future__ import unicode_literals import json import logging import re import time import requests.exceptions import six from six.moves.urllib import parse as urlparse from .. import MEDIA_TYPE_STIX_V20, MEDIA_TYPE_TAXII_V20 from ..common import ( _filter_kwargs_to_query_params, _grab_total_items_from_resource, _TAXIIEndpoint, _to_json ) from ..exceptions import AccessError, InvalidJSONError, ValidationError # Module-level logger log = logging.getLogger(__name__) def as_pages(func, start=0, per_request=0, *args, **kwargs): """Creates a generator for TAXII 2.0 endpoints that support pagination. Args: func (callable): A v20 function call that supports paged requests. Currently Get Objects and Get Manifest. start (int): The starting point for the page request. Default 0. per_request (int): How many items per request. Default 0. Use args or kwargs to pass filter information or other arguments required to make the call. """ resp = func(start=start, per_request=per_request, *args, **kwargs) yield _to_json(resp) total_obtained, total_available = _grab_total_items(resp) if total_available > per_request and total_obtained != per_request and total_obtained != float("inf"): log.warning("TAXII Server Response with different amount of objects! Setting per_request=%s", total_obtained) per_request = total_obtained start += per_request while start < total_available: resp = func(start=start, per_request=per_request, *args, **kwargs) yield _to_json(resp) total_in_request, total_available = _grab_total_items(resp) start += per_request def _grab_total_items(resp): """Extracts the total elements (from HTTP Header) available on the Endpoint making the request. This is specific to TAXII 2.0""" try: results = re.match(r"^items (\d+)-(\d+)/(\d+)$", resp.headers["Content-Range"]) if results: return int(results.group(2)) - int(results.group(1)) + 1, int(results.group(3)) results = re.match(r"^items (\d+)-(\d+)/\*$", resp.headers["Content-Range"]) if results: return int(results.group(2)) - int(results.group(1)) + 1, float("inf") results = re.match(r"^items \*/\*$", resp.headers["Content-Range"]) if results: return float("inf"), float("inf") results = re.match(r"^items \*/(\d+)$", resp.headers["Content-Range"]) if results: return float("inf"), int(results.group(1)) except (ValueError, IndexError) as e: six.raise_from(InvalidJSONError( "Invalid Content-Range was received from " + resp.request.url ), e) except KeyError: log.warning("TAXII Server Response did not include 'Content-Range' header - results could be incomplete.") return _grab_total_items_from_resource(resp), float("inf") class Status(_TAXIIEndpoint): """TAXII Status Resource. This class represents the ``Get Status`` endpoint (section 4.3) and also contains the information about the Status Resource (section 4.3.1) """ # We don't need to jump through the same lazy-load as with Collection, # since it's *far* less likely people will create these manually rather # than just getting them returned from Collection.add_objects(), and there # aren't other endpoints to call on the Status object. def __init__(self, url, conn=None, user=None, password=None, verify=True, proxies=None, status_info=None, auth=None, cert=None): """Create an API root resource endpoint. Args: url (str): URL of a TAXII status resource endpoint user (str): username for authentication (optional) password (str): password for authentication (optional) conn (_HTTPConnection): reuse connection object, as an alternative to providing username/password status_info (dict): Parsed JSON representing a response from the status endpoint, if already known. If not given, the endpoint will be queried. (optional) verify (bool): validate the entity credentials. (default: True) proxies (dict): key/value pair for http/https proxy settings. (optional) cert (str or tuple): SSL client certificate default, if String, path to ssl client cert file (.pem). If Tuple, (‘cert’, ‘key’) pair. (optional) """ super(Status, self).__init__(url, conn, user, password, verify, proxies, auth=auth, cert=cert) self.__raw = None if status_info: self._populate_fields(**status_info) self.__raw = status_info else: self.refresh() def __nonzero__(self): return self.status == "complete" __bool__ = __nonzero__ @property def _raw(self): """Get the "raw" status response (parsed JSON).""" return self.__raw @property def custom_properties(self): return self._custom_properties def refresh(self, accept=MEDIA_TYPE_TAXII_V20): """Updates Status information""" response = self.__raw = self._conn.get(self.url, headers={"Accept": accept}) self._populate_fields(**response) def wait_until_final(self, poll_interval=1, timeout=60): """It will poll the URL to grab the latest status resource in a given timeout and time interval. Args: poll_interval (int): how often to poll the status service. timeout (int): how long to poll the URL until giving up. Use <= 0 to wait forever """ start_time = time.time() elapsed = 0 while (self.status != "complete" and (timeout <= 0 or elapsed < timeout)): time.sleep(poll_interval) self.refresh() elapsed = time.time() - start_time def _populate_fields(self, id=None, status=None, total_count=None, success_count=None, failure_count=None, pending_count=None, request_timestamp=None, successes=None, failures=None, pendings=None, **kwargs): self.id = id # required self.status = status # required self.request_timestamp = request_timestamp # optional self.total_count = total_count # required self.success_count = success_count # required self.failure_count = failure_count # required self.pending_count = pending_count # required self.successes = successes or [] # optional self.failures = failures or [] # optional self.pendings = pendings or [] # optional # Anything not captured by the optional arguments is treated as custom self._custom_properties = kwargs self._validate_status() def _validate_status(self): """Validates Status information. Raises errors for required properties.""" if not self.id: msg = "No 'id' in Status for request '{}'" raise ValidationError(msg.format(self.url)) if not self.status: msg = "No 'status' in Status for request '{}'" raise ValidationError(msg.format(self.url)) if self.total_count is None: msg = "No 'total_count' in Status for request '{}'" raise ValidationError(msg.format(self.url)) if self.success_count is None: msg = "No 'success_count' in Status for request '{}'" raise ValidationError(msg.format(self.url)) if self.failure_count is None: msg = "No 'failure_count' in Status for request '{}'" raise ValidationError(msg.format(self.url)) if self.pending_count is None: msg = "No 'pending_count' in Status for request '{}'" raise ValidationError(msg.format(self.url)) if self.successes and len(self.successes) != self.success_count: msg = "Found successes={}, but success_count={} in status '{}'" raise ValidationError(msg.format(self.successes, self.success_count, self.id)) if self.pendings and len(self.pendings) != self.pending_count: msg = "Found pendings={}, but pending_count={} in status '{}'" raise ValidationError(msg.format(self.pendings, self.pending_count, self.id)) if self.failures and len(self.failures) != self.failure_count: msg = "Found failures={}, but failure_count={} in status '{}'" raise ValidationError(msg.format(self.failures, self.failure_count, self.id)) if (self.success_count + self.pending_count + self.failure_count != self.total_count): msg = ("(success_count={} + pending_count={} + " "failure_count={}) != total_count={} in status '{}'") raise ValidationError(msg.format(self.success_count, self.pending_count, self.failure_count, self.total_count, self.id)) class Collection(_TAXIIEndpoint): """Information about a TAXII Collection. This class represents the ``Get a Collection`` endpoint (section 5.2), and contains the information returned in the ``Collection Resource`` (section 5.2.1). Methods on this class can be used to invoke the following endpoints: - ``Get Objects`` (section 5.3) - ``Add Objects`` (section 5.4) - ``Get an Object`` (section 5.5) - ``Get Object Manifests`` (section 5.6) As obtained from an ApiRoot, an instance of this class shares connection(s) with all other collections obtained from the same ApiRoot, as well as the ApiRoot instance itself. Closing one will close them all. If this is undesirable, you may manually create Collection instances. """ def __init__(self, url, conn=None, user=None, password=None, verify=True, proxies=None, collection_info=None, auth=None, cert=None): """ Initialize a new Collection. Either user/password or conn may be given, but not both. The latter is intended for internal use, when sharing connection pools with an ApiRoot, mocking a connection for testing, etc. Users should use user/password (if required) which will create a new connection. Args: url (str): A TAXII endpoint for a collection user (str): User name for authentication (optional) password (str): Password for authentication (optional) verify (bool): Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use. Defaults to `True` (optional) conn (_HTTPConnection): A connection to reuse (optional) collection_info: Collection metadata, if known in advance (optional) verify (bool): validate the entity credentials. (default: True) proxies (dict): key/value pair for http/https proxy settings. (optional) cert (str or tuple): SSL client certificate default, if String, path to ssl client cert file (.pem). If Tuple, (‘cert’, ‘key’) pair. (optional) """ super(Collection, self).__init__(url, conn, user, password, verify, proxies, auth=auth, cert=cert) self._loaded = False self.__raw = None # Since the API Root "Get Collections" endpoint returns information on # all collections as a list, it's possible that we can create multiple # Collection objects from a single HTTPS request, and not need to call # `refresh` for each one. if collection_info: self._populate_fields(**collection_info) self.__raw = collection_info self._loaded = True @property def id(self): self._ensure_loaded() return self._id @property def title(self): self._ensure_loaded() return self._title @property def description(self): self._ensure_loaded() return self._description @property def can_read(self): self._ensure_loaded() return self._can_read @property def can_write(self): self._ensure_loaded() return self._can_write @property def media_types(self): self._ensure_loaded() return self._media_types @property def custom_properties(self): self._ensure_loaded() return self._custom_properties @property def objects_url(self): return self.url + "objects/" @property def manifest_url(self): return self.url + "manifest/" @property def _raw(self): """Get the "raw" collection information response (parsed JSON).""" self._ensure_loaded() return self.__raw def _populate_fields(self, id=None, title=None, description=None, can_read=None, can_write=None, media_types=None, **kwargs): self._id = id # required self._title = title # required self._description = description # optional self._can_read = can_read # required self._can_write = can_write # required self._media_types = media_types or [] # optional # Anything not captured by the optional arguments is treated as custom self._custom_properties = kwargs self._validate_collection() def _validate_collection(self): """Validates Collection information. Raises errors for required properties.""" if not self._id: msg = "No 'id' in Collection for request '{}'" raise ValidationError(msg.format(self.url)) if not self._title: msg = "No 'title' in Collection for request '{}'" raise ValidationError(msg.format(self.url)) if self._can_read is None: msg = "No 'can_read' in Collection for request '{}'" raise ValidationError(msg.format(self.url)) if self._can_write is None: msg = "No 'can_write' in Collection for request '{}'" raise ValidationError(msg.format(self.url)) if self._id not in self.url: msg = "The collection '{}' does not match the url for queries '{}'" raise ValidationError(msg.format(self._id, self.url)) def _ensure_loaded(self): if not self._loaded: self.refresh() def _verify_can_read(self): if not self.can_read: msg = "Collection '{}' does not allow reading." raise AccessError(msg.format(self.url)) def _verify_can_write(self): if not self.can_write: msg = "Collection '{}' does not allow writing." raise AccessError(msg.format(self.url)) def refresh(self, accept=MEDIA_TYPE_TAXII_V20): """Update Collection information""" response = self.__raw = self._conn.get( self.url, headers={"Accept": accept} ) self._populate_fields(**response) self._loaded = True def get_objects(self, accept=MEDIA_TYPE_STIX_V20, start=0, per_request=0, **filter_kwargs): """Implement the ``Get Objects`` endpoint (section 5.3). For pagination requests use ``as_pages`` method.""" self._verify_can_read() query_params = _filter_kwargs_to_query_params(filter_kwargs) headers = {"Accept": accept} if per_request > 0: headers["Range"] = "items={}-{}".format(start, (start + per_request) - 1) try: response = self._conn.get(self.objects_url, headers=headers, params=query_params) except requests.exceptions.HTTPError as e: if per_request > 0: # This is believed to be an error in TAXII 2.0 # http://docs.oasis-open.org/cti/taxii/v2.0/cs01/taxii-v2.0-cs01.html#_Toc496542716 headers["Range"] = "items {}-{}".format(start, (start + per_request) - 1) response = self._conn.get(self.objects_url, headers=headers, params=query_params) else: raise e return response def get_object(self, obj_id, version=None, accept=MEDIA_TYPE_STIX_V20): """Implement the ``Get an Object`` endpoint (section 5.5)""" self._verify_can_read() url = self.objects_url + str(obj_id) + "/" query_params = None if version: query_params = _filter_kwargs_to_query_params({"version": version}) return self._conn.get(url, headers={"Accept": accept}, params=query_params) def add_objects(self, bundle, wait_for_completion=True, poll_interval=1, timeout=60, accept=MEDIA_TYPE_TAXII_V20, content_type=MEDIA_TYPE_STIX_V20): """Implement the ``Add Objects`` endpoint (section 5.4) Add objects to the collection. This may be performed either synchronously or asynchronously. To add asynchronously, set wait_for_completion to False. If False, the latter two args are unused. If the caller wishes to monitor the status of the addition, it may do so in its own way. To add synchronously, set wait_for_completion to True, and optionally set the poll and timeout intervals. After initiating the addition, the caller will block, and the TAXII "status" service will be polled until the timeout expires, or the operation completes. Args: bundle: A STIX bundle with the objects to add (string, dict, binary) wait_for_completion (bool): Whether to wait for the add operation to complete before returning poll_interval (int): If waiting for completion, how often to poll the status service (seconds) timeout (int): If waiting for completion, how long to poll until giving up (seconds). Use <= 0 to wait forever accept (str): media type to include in the ``Accept:`` header. content_type (str): media type to include in the ``Content-Type:`` header. Returns: If ``wait_for_completion`` is False, a Status object corresponding to the initial status data returned from the service, is returned. The status may not yet be complete at this point. If ``wait_for_completion`` is True, a Status object corresponding to the completed operation is returned if it didn't time out; otherwise a Status object corresponding to the most recent data obtained before the timeout, is returned. """ self._verify_can_write() headers = { "Accept": accept, "Content-Type": content_type, } if isinstance(bundle, dict): json_text = json.dumps(bundle, ensure_ascii=False) data = json_text.encode("utf-8") elif isinstance(bundle, six.text_type): data = bundle.encode("utf-8") elif isinstance(bundle, six.binary_type): data = bundle else: raise TypeError("Don't know how to handle type '{}'".format( type(bundle).__name__)) status_json = self._conn.post(self.objects_url, headers=headers, data=data) status_url = urlparse.urljoin( self.url, "../../status/{}".format(status_json["id"]) ) status = Status(url=status_url, conn=self._conn, status_info=status_json) if not wait_for_completion or status.status == "complete": return status status.wait_until_final(poll_interval, timeout) return status def get_manifest(self, accept=MEDIA_TYPE_TAXII_V20, start=0, per_request=0, **filter_kwargs): """Implement the ``Get Object Manifests`` endpoint (section 5.6). For pagination requests use ``as_pages`` method.""" self._verify_can_read() query_params = _filter_kwargs_to_query_params(filter_kwargs) headers = {"Accept": accept} if per_request > 0: headers["Range"] = "items={}-{}".format(start, (start + per_request) - 1) try: response = self._conn.get(self.manifest_url, headers=headers, params=query_params) except requests.exceptions.HTTPError as e: if per_request > 0: # This is believed to be an error in TAXII 2.0 # http://docs.oasis-open.org/cti/taxii/v2.0/cs01/taxii-v2.0-cs01.html#_Toc496542716 headers["Range"] = "items {}-{}".format(start, (start + per_request) - 1) response = self._conn.get(self.manifest_url, headers=headers, params=query_params) else: raise e return response class ApiRoot(_TAXIIEndpoint): """Information about a TAXII API Root. This class corresponds to the ``Get API Root Information`` (section 4.2) and ``Get Collections`` (section 5.1) endpoints, and contains the information found in the corresponding ``API Root Resource`` (section 4.2.1) and ``Collections Resource`` (section 5.1.1). As obtained from a Server, each ApiRoot instance gets its own connection pool(s). Collections returned by instances of this class share the same pools as the instance, so closing one closes all. Also, the same username/password is used to connect to them, as was used for this ApiRoot. If either of these is undesirable, Collection instances may be created manually. """ def __init__(self, url, conn=None, user=None, password=None, verify=True, proxies=None, auth=None, cert=None): """Create an API root resource endpoint. Args: url (str): URL of a TAXII API root resource endpoint user (str): username for authentication (optional) password (str): password for authentication (optional) conn (_HTTPConnection): reuse connection object, as an alternative to providing username/password verify (bool): validate the entity credentials. (default: True) proxies (dict): key/value pair for http/https proxy settings. (optional) cert (str or tuple): SSL client certificate default, if String, path to ssl client cert file (.pem). If Tuple, (‘cert’, ‘key’) pair. (optional) """ super(ApiRoot, self).__init__(url, conn, user, password, verify, proxies, auth=auth, cert=cert) self._loaded_collections = False self._loaded_information = False self.__raw = None @property def collections(self): if not self._loaded_collections: self.refresh_collections() return self._collections @property def title(self): self._ensure_loaded_information() return self._title @property def description(self): self._ensure_loaded_information() return self._description @property def versions(self): self._ensure_loaded_information() return self._versions @property def max_content_length(self): self._ensure_loaded_information() return self._max_content_length @property def custom_properties(self): self._ensure_loaded_information() return self._custom_properties @property def _raw(self): """Get the "raw" API root information response (parsed JSON).""" self._ensure_loaded_information() return self.__raw def _ensure_loaded_information(self): if not self._loaded_information: self.refresh_information() def _validate_api_root(self): """Validates API Root information. Raises errors for required properties.""" if not self._title: msg = "No 'title' in API Root for request '{}'" raise ValidationError(msg.format(self.url)) if not self._versions: msg = "No 'versions' in API Root for request '{}'" raise ValidationError(msg.format(self.url)) if self._max_content_length is None: msg = "No 'max_content_length' in API Root for request '{}'" raise ValidationError(msg.format(self.url)) def _populate_fields(self, title=None, description=None, versions=None, max_content_length=None, **kwargs): self._title = title # required self._description = description # optional self._versions = versions or [] # required self._max_content_length = max_content_length # required # Anything not captured by the optional arguments is treated as custom self._custom_properties = kwargs self._validate_api_root() def refresh(self, accept=MEDIA_TYPE_TAXII_V20): """Update the API Root's information and list of Collections""" self.refresh_information(accept) self.refresh_collections(accept) def refresh_information(self, accept=MEDIA_TYPE_TAXII_V20): """Update the properties of this API Root. This invokes the ``Get API Root Information`` endpoint. """ response = self.__raw = self._conn.get(self.url, headers={"Accept": accept}) self._populate_fields(**response) self._loaded_information = True def refresh_collections(self, accept=MEDIA_TYPE_TAXII_V20): """Update the list of Collections contained by this API Root. This invokes the ``Get Collections`` endpoint. """ url = self.url + "collections/" response = self._conn.get(url, headers={"Accept": accept}) self._collections = [] for item in response.get("collections", []): # optional collection_url = url + item["id"] + "/" collection = Collection(collection_url, conn=self._conn, collection_info=item) self._collections.append(collection) self._loaded_collections = True def get_status(self, status_id, accept=MEDIA_TYPE_TAXII_V20): status_url = self.url + "status/" + status_id + "/" response = self._conn.get(status_url, headers={"Accept": accept}) return Status(status_url, conn=self._conn, status_info=response) class Server(_TAXIIEndpoint): """Information about a server hosting a Discovery service. This class corresponds to the Server Discovery endpoint (section 4.1) and the Discovery Resource returned from that endpoint (section 4.1.1). ApiRoot instances obtained from an instance of this class are created with the same username/password as was used in this instance. If that's incorrect, an ApiRoot instance may be created directly with the desired username and password. Also, they use separate connection pools so that they can be independent: closing one won't close others, and closing this server object won't close any of the ApiRoot objects (which may refer to different hosts than was used for discovery). """ def __init__(self, url, conn=None, user=None, password=None, verify=True, proxies=None, auth=None, cert=None): """Create a server discovery endpoint. Args: url (str): URL of a TAXII server discovery endpoint user (str): username for authentication (optional) password (str): password for authentication (optional) conn (_HTTPConnection): reuse connection object, as an alternative to providing username/password verify (bool): validate the entity credentials. (default: True) proxies (dict): key/value pair for http/https proxy settings. (optional) cert (str or tuple): SSL client certificate default, if String, path to ssl client cert file (.pem). If Tuple, (‘cert’, ‘key’) pair. (optional) """ super(Server, self).__init__(url, conn, user, password, verify, proxies, auth=auth, cert=cert) self._user = user self._password = password self._verify = verify self._proxies = proxies self._loaded = False self.__raw = None self._auth = auth self._cert = cert @property def title(self): self._ensure_loaded() return self._title @property def description(self): self._ensure_loaded() return self._description @property def contact(self): self._ensure_loaded() return self._contact @property def default(self): self._ensure_loaded() return self._default @property def api_roots(self): self._ensure_loaded() return self._api_roots @property def custom_properties(self): self._ensure_loaded() return self._custom_properties @property def _raw(self): """Get the "raw" server discovery response (parsed JSON).""" self._ensure_loaded() return self.__raw def _ensure_loaded(self): if not self._loaded: self.refresh() def _validate_server(self): """Validates server information. Raises errors for required properties. """ if not self._title: msg = "No 'title' in Server Discovery for request '{}'" raise ValidationError(msg.format(self.url)) def _populate_fields(self, title=None, description=None, contact=None, api_roots=None, default=None, **kwargs): self._title = title # required self._description = description # optional self._contact = contact # optional roots = api_roots or [] # optional self._api_roots = [ApiRoot(url, user=self._user, password=self._password, verify=self._verify, proxies=self._proxies, auth=self._auth, cert=self._cert) for url in roots] # If 'default' is one of the existing API Roots, reuse that object # rather than creating a duplicate. The TAXII 2.0 spec says that the # `default` API Root MUST be an item in `api_roots`. root_dict = dict(zip(roots, self._api_roots)) self._default = root_dict.get(default) # optional # Anything not captured by the optional arguments is treated as custom self._custom_properties = kwargs self._validate_server() def refresh(self): """Update the Server information and list of API Roots""" response = self.__raw = self._conn.get(self.url) self._populate_fields(**response) self._loaded = True cti-taxii-client-2.3.0/taxii2client/v21/000077500000000000000000000000001402275443100176745ustar00rootroot00000000000000cti-taxii-client-2.3.0/taxii2client/v21/__init__.py000066400000000000000000000714751402275443100220230ustar00rootroot00000000000000"""Python TAXII 2.1 Client API""" from __future__ import unicode_literals import json import logging import time import six from six.moves.urllib import parse as urlparse from .. import MEDIA_TYPE_TAXII_V21 from ..common import ( _filter_kwargs_to_query_params, _grab_total_items_from_resource, _TAXIIEndpoint ) from ..exceptions import AccessError, ValidationError # Module-level logger log = logging.getLogger(__name__) def as_pages(func, per_request=0, *args, **kwargs): """Creates a generator for TAXII 2.1 endpoints that support pagination. Args: func (callable): A v21 function that supports paged requests. Currently Get Objects and Get Manifest. per_request (int): How many items per request. Default 0. Use args or kwargs to pass filter information or other arguments required to make the call. """ envelope = func(limit=per_request, *args, **kwargs) yield envelope total_obtained = _grab_total_items_from_resource(envelope) if envelope.get("more", False) and total_obtained != per_request: log.warning("TAXII Server Response with different amount of objects! Setting limit=%s", total_obtained) per_request = total_obtained # The while loop will not be executed if the response is received in full. while envelope.get("more", False): envelope = func(limit=per_request, next=envelope.get("next", ""), *args, **kwargs) yield envelope class Status(_TAXIIEndpoint): """TAXII Status Resource. This class represents the ``Get Status`` endpoint (section 4.3) and also contains the information about the Status Resource (section 4.3.1) """ # We don't need to jump through the same lazy-load as with Collection, # since it's *far* less likely people will create these manually rather # than just getting them returned from Collection.add_objects(), and there # aren't other endpoints to call on the Status object. def __init__(self, url, conn=None, user=None, password=None, verify=True, proxies=None, status_info=None, auth=None, cert=None): """Create an API root resource endpoint. Args: url (str): URL of a TAXII status resource endpoint user (str): username for authentication (optional) password (str): password for authentication (optional) conn (_HTTPConnection): reuse connection object, as an alternative to providing username/password status_info (dict): Parsed JSON representing a response from the status endpoint, if already known. If not given, the endpoint will be queried. (optional) verify (bool): validate the entity credentials. (default: True) proxies (dict): key/value pair for http/https proxy settings. (optional) cert (str or tuple): SSL client certificate default, if String, path to ssl client cert file (.pem). If Tuple, (‘cert’, ‘key’) pair. (optional) """ super(Status, self).__init__(url, conn, user, password, verify, proxies, "2.1", auth=auth, cert=cert) self.__raw = None if status_info: self._populate_fields(**status_info) self.__raw = status_info else: self.refresh() def __nonzero__(self): return self.status == "complete" __bool__ = __nonzero__ @property def _raw(self): """Get the "raw" status response (parsed JSON).""" return self.__raw @property def custom_properties(self): return self._custom_properties def refresh(self, accept=MEDIA_TYPE_TAXII_V21): """Updates Status information""" response = self.__raw = self._conn.get(self.url, headers={"Accept": accept}) self._populate_fields(**response) def wait_until_final(self, poll_interval=1, timeout=60): """It will poll the URL to grab the latest status resource in a given timeout and time interval. Args: poll_interval (int): how often to poll the status service. timeout (int): how long to poll the URL until giving up. Use <= 0 to wait forever """ start_time = time.time() elapsed = 0 while (self.status != "complete" and (timeout <= 0 or elapsed < timeout)): time.sleep(poll_interval) self.refresh() elapsed = time.time() - start_time def _populate_fields(self, id=None, status=None, total_count=None, success_count=None, failure_count=None, pending_count=None, request_timestamp=None, successes=None, failures=None, pendings=None, **kwargs): self.id = id # required self.status = status # required self.request_timestamp = request_timestamp # optional self.total_count = total_count # required self.success_count = success_count # required self.failure_count = failure_count # required self.pending_count = pending_count # required self.successes = successes or [] # optional self.failures = failures or [] # optional self.pendings = pendings or [] # optional # Anything not captured by the optional arguments is treated as custom self._custom_properties = kwargs self._validate_status() def _validate_status(self): """Validates Status information. Raises errors for required properties.""" if not self.id: msg = "No 'id' in Status for request '{}'" raise ValidationError(msg.format(self.url)) if not self.status: msg = "No 'status' in Status for request '{}'" raise ValidationError(msg.format(self.url)) if self.total_count is None: msg = "No 'total_count' in Status for request '{}'" raise ValidationError(msg.format(self.url)) if self.success_count is None: msg = "No 'success_count' in Status for request '{}'" raise ValidationError(msg.format(self.url)) if self.failure_count is None: msg = "No 'failure_count' in Status for request '{}'" raise ValidationError(msg.format(self.url)) if self.pending_count is None: msg = "No 'pending_count' in Status for request '{}'" raise ValidationError(msg.format(self.url)) if self.successes and len(self.successes) != self.success_count: msg = "Found successes={}, but success_count={} in status '{}'" raise ValidationError(msg.format(self.successes, self.success_count, self.id)) if self.pendings and len(self.pendings) != self.pending_count: msg = "Found pendings={}, but pending_count={} in status '{}'" raise ValidationError(msg.format(self.pendings, self.pending_count, self.id)) if self.failures and len(self.failures) != self.failure_count: msg = "Found failures={}, but failure_count={} in status '{}'" raise ValidationError(msg.format(self.failures, self.failure_count, self.id)) if (self.success_count + self.pending_count + self.failure_count != self.total_count): msg = ("(success_count={} + pending_count={} + " "failure_count={}) != total_count={} in status '{}'") raise ValidationError(msg.format(self.success_count, self.pending_count, self.failure_count, self.total_count, self.id)) class Collection(_TAXIIEndpoint): """Information about a TAXII Collection. This class represents the ``Get a Collection`` endpoint (section 5.2), and contains the information returned in the ``Collection Resource`` (section 5.2.1). Methods on this class can be used to invoke the following endpoints: - ``Get Objects`` (section 5.3) - ``Add Objects`` (section 5.4) - ``Get an Object`` (section 5.5) - ``Get Object Manifests`` (section 5.6) As obtained from an ApiRoot, an instance of this class shares connection(s) with all other collections obtained from the same ApiRoot, as well as the ApiRoot instance itself. Closing one will close them all. If this is undesirable, you may manually create Collection instances. """ def __init__(self, url, conn=None, user=None, password=None, verify=True, proxies=None, collection_info=None, auth=None, cert=None): """ Initialize a new Collection. Either user/password or conn may be given, but not both. The latter is intended for internal use, when sharing connection pools with an ApiRoot, mocking a connection for testing, etc. Users should use user/password (if required) which will create a new connection. Args: url (str): A TAXII endpoint for a collection user (str): User name for authentication (optional) password (str): Password for authentication (optional) verify (bool): Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use. Defaults to `True` (optional) conn (_HTTPConnection): A connection to reuse (optional) collection_info: Collection metadata, if known in advance (optional) verify (bool): validate the entity credentials. (default: True) proxies (dict): key/value pair for http/https proxy settings. (optional) cert (str or tuple): SSL client certificate default, if String, path to ssl client cert file (.pem). If Tuple, (‘cert’, ‘key’) pair. (optional) """ super(Collection, self).__init__(url, conn, user, password, verify, proxies, "2.1", auth=auth, cert=cert) self._loaded = False self.__raw = None # Since the API Root "Get Collections" endpoint returns information on # all collections as a list, it's possible that we can create multiple # Collection objects from a single HTTPS request, and not need to call # `refresh` for each one. if collection_info: self._populate_fields(**collection_info) self.__raw = collection_info self._loaded = True @property def id(self): self._ensure_loaded() return self._id @property def title(self): self._ensure_loaded() return self._title @property def description(self): self._ensure_loaded() return self._description @property def alias(self): self._ensure_loaded() return self._alias @property def can_read(self): self._ensure_loaded() return self._can_read @property def can_write(self): self._ensure_loaded() return self._can_write @property def media_types(self): self._ensure_loaded() return self._media_types @property def custom_properties(self): self._ensure_loaded() return self._custom_properties @property def objects_url(self): return self.url + "objects/" @property def _raw(self): """Get the "raw" collection information response (parsed JSON).""" self._ensure_loaded() return self.__raw def _populate_fields(self, id=None, title=None, description=None, alias=None, can_read=None, can_write=None, media_types=None, **kwargs): self._id = id # required self._title = title # required self._description = description # optional self._alias = alias # optional self._can_read = can_read # required self._can_write = can_write # required self._media_types = media_types or [] # optional # Anything not captured by the optional arguments is treated as custom self._custom_properties = kwargs self._validate_collection() def _validate_collection(self): """Validates Collection information. Raises errors for required properties.""" if not self._id: msg = "No 'id' in Collection for request '{}'" raise ValidationError(msg.format(self.url)) if not self._title: msg = "No 'title' in Collection for request '{}'" raise ValidationError(msg.format(self.url)) if self._can_read is None: msg = "No 'can_read' in Collection for request '{}'" raise ValidationError(msg.format(self.url)) if self._can_write is None: msg = "No 'can_write' in Collection for request '{}'" raise ValidationError(msg.format(self.url)) if self._id not in self.url: msg = "The collection '{}' does not match the url for queries '{}'" raise ValidationError(msg.format(self._id, self.url)) def _ensure_loaded(self): if not self._loaded: self.refresh() def _verify_can_read(self): if not self.can_read: msg = "Collection '{}' does not allow reading." raise AccessError(msg.format(self.url)) def _verify_can_write(self): if not self.can_write: msg = "Collection '{}' does not allow writing." raise AccessError(msg.format(self.url)) def refresh(self, accept=MEDIA_TYPE_TAXII_V21): """Update Collection information""" response = self.__raw = self._conn.get(self.url, headers={"Accept": accept}) self._populate_fields(**response) self._loaded = True def get_objects(self, accept=MEDIA_TYPE_TAXII_V21, **filter_kwargs): """Implement the ``Get Objects`` endpoint (section 5.3)""" self._verify_can_read() query_params = _filter_kwargs_to_query_params(filter_kwargs) return self._conn.get(self.objects_url, headers={"Accept": accept}, params=query_params) def get_object(self, obj_id, accept=MEDIA_TYPE_TAXII_V21, **filter_kwargs): """Implement the ``Get an Object`` endpoint (section 5.5)""" self._verify_can_read() url = self.objects_url + str(obj_id) + "/" query_params = _filter_kwargs_to_query_params(filter_kwargs) return self._conn.get(url, headers={"Accept": accept}, params=query_params) def delete_object(self, obj_id, accept=MEDIA_TYPE_TAXII_V21, **filter_kwargs): """Implement the ``Delete an Object`` endpoint (section 5.7)""" self._verify_can_write() url = self.objects_url + str(obj_id) + "/" query_params = _filter_kwargs_to_query_params(filter_kwargs) return self._conn.delete(url, headers={"Accept": accept}, params=query_params) def object_versions(self, obj_id, accept=MEDIA_TYPE_TAXII_V21, **filter_kwargs): """Implement the ``Get Object Versions`` endpoint (section 5.8)""" self._verify_can_read() url = self.objects_url + str(obj_id) + "/versions/" query_params = _filter_kwargs_to_query_params(filter_kwargs) return self._conn.get(url, headers={"Accept": accept}, params=query_params) def add_objects(self, envelope, wait_for_completion=True, poll_interval=1, timeout=60, accept=MEDIA_TYPE_TAXII_V21, content_type=MEDIA_TYPE_TAXII_V21): """Implement the ``Add Objects`` endpoint (section 5.4) Add objects to the collection. This may be performed either synchronously or asynchronously. To add asynchronously, set wait_for_completion to False. If False, the latter two args are unused. If the caller wishes to monitor the status of the addition, it may do so in its own way. To add synchronously, set wait_for_completion to True, and optionally set the poll and timeout intervals. After initiating the addition, the caller will block, and the TAXII "status" service will be polled until the timeout expires, or the operation completes. Args: envelope: A TAXII envelope with the objects to add (string, dict, binary) wait_for_completion (bool): Whether to wait for the add operation to complete before returning poll_interval (int): If waiting for completion, how often to poll the status service (seconds) timeout (int): If waiting for completion, how long to poll until giving up (seconds). Use <= 0 to wait forever accept (str): media type to include in the ``Accept:`` header. content_type (str): media type to include in the ``Content-Type:`` header. Returns: If ``wait_for_completion`` is False, a Status object corresponding to the initial status data returned from the service, is returned. The status may not yet be complete at this point. If ``wait_for_completion`` is True, a Status object corresponding to the completed operation is returned if it didn't time out; otherwise a Status object corresponding to the most recent data obtained before the timeout, is returned. """ self._verify_can_write() headers = { "Accept": accept, "Content-Type": content_type, } if isinstance(envelope, dict): json_text = json.dumps(envelope, ensure_ascii=False) data = json_text.encode("utf-8") elif isinstance(envelope, six.text_type): data = envelope.encode("utf-8") elif isinstance(envelope, six.binary_type): data = envelope else: raise TypeError("Don't know how to handle type '{}'".format( type(envelope).__name__)) status_json = self._conn.post(self.objects_url, headers=headers, data=data) status_url = urlparse.urljoin( self.url, "../../status/{}".format(status_json["id"]) ) status = Status(url=status_url, conn=self._conn, status_info=status_json) if not wait_for_completion or status.status == "complete": return status status.wait_until_final(poll_interval, timeout) return status def get_manifest(self, accept=MEDIA_TYPE_TAXII_V21, **filter_kwargs): """Implement the ``Get Object Manifests`` endpoint (section 5.6).""" self._verify_can_read() query_params = _filter_kwargs_to_query_params(filter_kwargs) return self._conn.get(self.url + "manifest/", headers={"Accept": accept}, params=query_params) class ApiRoot(_TAXIIEndpoint): """Information about a TAXII API Root. This class corresponds to the ``Get API Root Information`` (section 4.2) and ``Get Collections`` (section 5.1) endpoints, and contains the information found in the corresponding ``API Root Resource`` (section 4.2.1) and ``Collections Resource`` (section 5.1.1). As obtained from a Server, each ApiRoot instance gets its own connection pool(s). Collections returned by instances of this class share the same pools as the instance, so closing one closes all. Also, the same username/password is used to connect to them, as was used for this ApiRoot. If either of these is undesirable, Collection instances may be created manually. """ def __init__(self, url, conn=None, user=None, password=None, verify=True, proxies=None, auth=None, cert=None): """Create an API root resource endpoint. Args: url (str): URL of a TAXII API root resource endpoint user (str): username for authentication (optional) password (str): password for authentication (optional) conn (_HTTPConnection): reuse connection object, as an alternative to providing username/password verify (bool): validate the entity credentials. (default: True) proxies (dict): key/value pair for http/https proxy settings. (optional) cert (str or tuple): SSL client certificate default, if String, path to ssl client cert file (.pem). If Tuple, (‘cert’, ‘key’) pair. (optional) """ super(ApiRoot, self).__init__(url, conn, user, password, verify, proxies, "2.1", auth=auth, cert=cert) self._loaded_collections = False self._loaded_information = False self.__raw = None @property def collections(self): if not self._loaded_collections: self.refresh_collections() return self._collections @property def title(self): self._ensure_loaded_information() return self._title @property def description(self): self._ensure_loaded_information() return self._description @property def versions(self): self._ensure_loaded_information() return self._versions @property def max_content_length(self): self._ensure_loaded_information() return self._max_content_length @property def custom_properties(self): self._ensure_loaded_information() return self._custom_properties @property def _raw(self): """Get the "raw" API root information response (parsed JSON).""" self._ensure_loaded_information() return self.__raw def _ensure_loaded_information(self): if not self._loaded_information: self.refresh_information() def _validate_api_root(self): """Validates API Root information. Raises errors for required properties.""" if not self._title: msg = "No 'title' in API Root for request '{}'" raise ValidationError(msg.format(self.url)) if not self._versions: msg = "No 'versions' in API Root for request '{}'" raise ValidationError(msg.format(self.url)) if self._max_content_length is None: msg = "No 'max_content_length' in API Root for request '{}'" raise ValidationError(msg.format(self.url)) def _populate_fields(self, title=None, description=None, versions=None, max_content_length=None, **kwargs): self._title = title # required self._description = description # optional self._versions = versions or [] # required self._max_content_length = max_content_length # required # Anything not captured by the optional arguments is treated as custom self._custom_properties = kwargs self._validate_api_root() def refresh(self, accept=MEDIA_TYPE_TAXII_V21): """Update the API Root's information and list of Collections""" self.refresh_information(accept) self.refresh_collections(accept) def refresh_information(self, accept=MEDIA_TYPE_TAXII_V21): """Update the properties of this API Root. This invokes the ``Get API Root Information`` endpoint. """ response = self.__raw = self._conn.get(self.url, headers={"Accept": accept}) self._populate_fields(**response) self._loaded_information = True def refresh_collections(self, accept=MEDIA_TYPE_TAXII_V21): """Update the list of Collections contained by this API Root. This invokes the ``Get Collections`` endpoint. """ url = self.url + "collections/" response = self._conn.get(url, headers={"Accept": accept}) self._collections = [] for item in response.get("collections", []): # optional collection_url = url + item["id"] + "/" collection = Collection(collection_url, conn=self._conn, collection_info=item) self._collections.append(collection) self._loaded_collections = True def get_status(self, status_id, accept=MEDIA_TYPE_TAXII_V21): status_url = self.url + "status/" + status_id + "/" response = self._conn.get(status_url, headers={"Accept": accept}) return Status(status_url, conn=self._conn, status_info=response) class Server(_TAXIIEndpoint): """Information about a server hosting a Discovery service. This class corresponds to the Server Discovery endpoint (section 4.1) and the Discovery Resource returned from that endpoint (section 4.1.1). ApiRoot instances obtained from an instance of this class are created with the same username/password as was used in this instance. If that's incorrect, an ApiRoot instance may be created directly with the desired username and password. Also, they use separate connection pools so that they can be independent: closing one won't close others, and closing this server object won't close any of the ApiRoot objects (which may refer to different hosts than was used for discovery). """ def __init__(self, url, conn=None, user=None, password=None, verify=True, proxies=None, auth=None, cert=None): """Create a server discovery endpoint. Args: url (str): URL of a TAXII server discovery endpoint user (str): username for authentication (optional) password (str): password for authentication (optional) conn (_HTTPConnection): reuse connection object, as an alternative to providing username/password verify (bool): validate the entity credentials. (default: True) proxies (dict): key/value pair for http/https proxy settings. (optional) cert (str or tuple): SSL client certificate default, if String, path to ssl client cert file (.pem). If Tuple, (‘cert’, ‘key’) pair. (optional) """ super(Server, self).__init__(url, conn, user, password, verify, proxies, "2.1", auth=auth, cert=cert) self._user = user self._password = password self._verify = verify self._proxies = proxies self._loaded = False self.__raw = None self._auth = auth self._cert = cert @property def title(self): self._ensure_loaded() return self._title @property def description(self): self._ensure_loaded() return self._description @property def contact(self): self._ensure_loaded() return self._contact @property def default(self): self._ensure_loaded() return self._default @property def api_roots(self): self._ensure_loaded() return self._api_roots @property def custom_properties(self): self._ensure_loaded() return self._custom_properties @property def _raw(self): """Get the "raw" server discovery response (parsed JSON).""" self._ensure_loaded() return self.__raw def _ensure_loaded(self): if not self._loaded: self.refresh() def _validate_server(self): """Validates server information. Raises errors for required properties. """ if not self._title: msg = "No 'title' in Server Discovery for request '{}'" raise ValidationError(msg.format(self.url)) def _populate_fields(self, title=None, description=None, contact=None, api_roots=None, default=None, **kwargs): self._title = title # required self._description = description # optional self._contact = contact # optional roots = api_roots or [] # optional self._api_roots = [ ApiRoot(urlparse.urljoin(self.url, url), user=self._user, password=self._password, verify=self._verify, proxies=self._proxies, auth=self._auth, cert=self._cert) for url in roots ] # If 'default' is one of the existing API Roots, reuse that object # rather than creating a duplicate. The TAXII 2.1 spec says that the # `default` API Root MUST be an item in `api_roots`. root_dict = dict(zip(roots, self._api_roots)) self._default = root_dict.get(default) # optional # Anything not captured by the optional arguments is treated as custom self._custom_properties = kwargs self._validate_server() def refresh(self): """Update the Server information and list of API Roots""" response = self.__raw = self._conn.get(self.url) self._populate_fields(**response) self._loaded = True cti-taxii-client-2.3.0/taxii2client/version.py000066400000000000000000000000261402275443100213210ustar00rootroot00000000000000__version__ = "2.3.0" cti-taxii-client-2.3.0/tox.ini000066400000000000000000000010411402275443100161740ustar00rootroot00000000000000[tox] envlist = py36,py37,py38,py39,packaging,pre-commit-check [testenv] deps = -U tox pytest pytest-cov coverage responses commands = pytest --cov=taxii2client taxii2client/test/ --cov-report term-missing passenv = GITHUB_* [testenv:packaging] deps = twine commands = python setup.py sdist bdist_wheel --universal twine check dist/* [testenv:pre-commit-check] deps = pre-commit commands = pre-commit run --all-files [gh-actions] python = 3.6: py36 3.7: py37 3.8: py38 3.9: py39, packaging, pre-commit-check