"""Configuration of Airflow Docs"""
from __future__ import annotations
import json
import os
import pathlib
import sys
from collections import defaultdict
from pathlib import Path
from typing import Any
import yaml
import airflow
from airflow.configuration import AirflowConfigParser, default_config_yaml
sys.path.append(str(Path(__file__).parent / "exts"))
from docs_build.third_party_inventories import THIRD_PARTY_INDEXES
CONF_DIR = pathlib.Path(__file__).parent.absolute()
INVENTORY_CACHE_DIR = CONF_DIR / "_inventory_cache"
ROOT_DIR = CONF_DIR.parent
FOR_PRODUCTION = os.environ.get("AIRFLOW_FOR_PRODUCTION", "false") == "true"
PACKAGE_NAME = os.environ.get("AIRFLOW_PACKAGE_NAME", "apache-airflow")
PACKAGE_DIR: pathlib.Path
if PACKAGE_NAME == "apache-airflow":
PACKAGE_DIR = ROOT_DIR / "airflow"
PACKAGE_VERSION = airflow.__version__
SYSTEM_TESTS_DIR = None
elif PACKAGE_NAME.startswith("apache-airflow-providers-"):
from provider_yaml_utils import load_package_data
ALL_PROVIDER_YAMLS = load_package_data()
try:
CURRENT_PROVIDER = next(
provider_yaml
for provider_yaml in ALL_PROVIDER_YAMLS
if provider_yaml["package-name"] == PACKAGE_NAME
)
except StopIteration:
raise Exception(f"Could not find provider.yaml file for package: {PACKAGE_NAME}")
PACKAGE_DIR = pathlib.Path(CURRENT_PROVIDER["package-dir"])
PACKAGE_VERSION = CURRENT_PROVIDER["versions"][0]
SYSTEM_TESTS_DIR = CURRENT_PROVIDER["system-tests-dir"]
elif PACKAGE_NAME == "apache-airflow-providers":
from provider_yaml_utils import load_package_data
PACKAGE_DIR = ROOT_DIR / "airflow" / "providers"
PACKAGE_VERSION = "devel"
ALL_PROVIDER_YAMLS = load_package_data()
SYSTEM_TESTS_DIR = None
elif PACKAGE_NAME == "helm-chart":
PACKAGE_DIR = ROOT_DIR / "chart"
chart_yaml_file = PACKAGE_DIR / "Chart.yaml"
with chart_yaml_file.open() as chart_file:
chart_yaml_contents = yaml.safe_load(chart_file)
PACKAGE_VERSION = chart_yaml_contents["version"]
SYSTEM_TESTS_DIR = None
else:
PACKAGE_VERSION = "devel"
SYSTEM_TESTS_DIR = None
os.environ["AIRFLOW_PACKAGE_NAME"] = PACKAGE_NAME
os.environ["BUILDING_AIRFLOW_DOCS"] = "TRUE"
project = PACKAGE_NAME
version = PACKAGE_VERSION
release = PACKAGE_VERSION
rst_epilog = f"""
.. |version| replace:: {version}
.. |airflow-version| replace:: {airflow.__version__}
.. |experimental| replace:: This is an :ref:`experimental feature <experimental>`.
"""
smartquotes_excludes = {"builders": ["man", "text", "spelling"]}
extensions = [
"provider_init_hack",
"sphinx.ext.autodoc",
"sphinx.ext.viewcode",
"sphinxarg.ext",
"sphinx.ext.intersphinx",
"exampleinclude",
"docroles",
"removemarktransform",
"sphinx_copybutton",
"airflow_intersphinx",
"sphinxcontrib.spelling",
"sphinx_airflow_theme",
"redirects",
"substitution_extensions",
]
if PACKAGE_NAME == "apache-airflow":
extensions.extend(
[
"sphinx_jinja",
"sphinx.ext.graphviz",
"sphinxcontrib.httpdomain",
"sphinxcontrib.httpdomain",
"extra_files_with_substitutions",
"sphinxcontrib.redoc",
"sphinx_script_update",
]
)
if PACKAGE_NAME == "apache-airflow-providers":
extensions.extend(
[
"sphinx_jinja",
"operators_and_hooks_ref",
"providers_packages_ref",
]
)
elif PACKAGE_NAME == "helm-chart":
extensions.append("sphinx_jinja")
elif PACKAGE_NAME == "docker-stack":
pass
elif PACKAGE_NAME.startswith("apache-airflow-providers-"):
extensions.extend(
[
"extra_provider_files_with_substitutions",
"autoapi.extension",
]
)
else:
extensions.append("autoapi.extension")
exclude_patterns: list[str]
if PACKAGE_NAME == "apache-airflow":
exclude_patterns = [
"_api/airflow/index.rst",
"_api/airflow/policies/index.rst",
"README.rst",
]
elif PACKAGE_NAME.startswith("apache-airflow-providers-"):
extensions.extend(
[
"sphinx_jinja",
]
)
exclude_patterns = ["operators/_partials"]
else:
exclude_patterns = []
def _get_rst_filepath_from_path(filepath: pathlib.Path):
if filepath.is_dir():
result = filepath
else:
if filepath.name == "__init__.py":
result = filepath.parent
else:
result = filepath.with_name(filepath.stem)
result /= "index.rst"
return f"_api/{result.relative_to(ROOT_DIR)}"
if PACKAGE_NAME == "apache-airflow":
_allowed_top_level = ("exceptions.py", "policies.py")
browsable_packages = {
"hooks",
"decorators",
"example_dags",
"executors",
"operators",
"providers",
"secrets",
"sensors",
"timetables",
"triggers",
"utils",
}
browsable_utils: set[str] = set()
root = ROOT_DIR / "airflow"
for path in root.iterdir():
if path.is_file() and path.name not in _allowed_top_level:
exclude_patterns.append(_get_rst_filepath_from_path(path))
if path.is_dir() and path.name not in browsable_packages:
exclude_patterns.append(f"_api/airflow/{path.name}")
for path in (root / "utils").iterdir():
if path.name not in browsable_utils:
exclude_patterns.append(_get_rst_filepath_from_path(path))
elif PACKAGE_NAME != "docker-stack":
exclude_patterns.extend(
_get_rst_filepath_from_path(f) for f in pathlib.Path(PACKAGE_DIR).glob("**/example_dags")
)
templates_path = ["templates"]
keep_warnings = True
html_theme = "sphinx_airflow_theme"
if PACKAGE_NAME == "apache-airflow":
html_title = "Airflow Documentation"
else:
html_title = f"{PACKAGE_NAME} Documentation"
html_short_title = ""
html_favicon = "../airflow/www/static/pin_32.png"
if PACKAGE_NAME in ["apache-airflow", "helm-chart"]:
html_static_path = [f"{PACKAGE_NAME}/static"]
else:
html_static_path = []
if PACKAGE_NAME in ["apache-airflow", "helm-chart"]:
html_js_files = ["gh-jira-links.js"]
else:
html_js_files = []
if PACKAGE_NAME == "apache-airflow":
html_extra_path = [
f"{ROOT_DIR}/docs/apache-airflow/howto/docker-compose/airflow.sh",
]
html_extra_with_substitutions = [
f"{ROOT_DIR}/docs/apache-airflow/howto/docker-compose/docker-compose.yaml",
f"{ROOT_DIR}/docs/docker-stack/build.rst",
]
manual_substitutions_in_generated_html = [
"installation/installing-from-pypi.html",
"installation/installing-from-sources.html",
]
if PACKAGE_NAME.startswith("apache-airflow-providers"):
manual_substitutions_in_generated_html = ["example-dags.html", "operators.html", "index.html"]
if PACKAGE_NAME == "docker-stack":
manual_substitutions_in_generated_html = ["build.html"]
html_sidebars = {
"**": [
"version-selector.html",
"searchbox.html",
"globaltoc.html",
]
if FOR_PRODUCTION and PACKAGE_VERSION != "devel"
else [
"searchbox.html",
"globaltoc.html",
]
}
html_use_index = True
html_show_copyright = False
if PACKAGE_NAME.startswith("apache-airflow-providers-"):
html_theme_options: dict[str, Any] = {"hide_website_buttons": True, "sidebar_includehidden": False}
else:
html_theme_options = {"hide_website_buttons": True, "sidebar_includehidden": True}
if FOR_PRODUCTION:
html_theme_options["navbar_links"] = [
{"href": "/community/", "text": "Community"},
{"href": "/meetups/", "text": "Meetups"},
{"href": "/docs/", "text": "Documentation"},
{"href": "/use-cases/", "text": "Use-cases"},
{"href": "/announcements/", "text": "Announcements"},
{"href": "/blog/", "text": "Blog"},
{"href": "/ecosystem/", "text": "Ecosystem"},
]
html_context = {
"theme_analytics_id": "UA-140539454-1",
"theme_vcs_pageview_mode": "edit",
"conf_py_path": f"/docs/{PACKAGE_NAME}/",
"github_user": "apache",
"github_repo": "airflow",
"github_version": "main",
"display_github": "main",
"suffix": ".rst",
}
if PACKAGE_NAME == "apache-airflow":
deprecated_options: dict[str, dict[str, tuple[str, str, str]]] = defaultdict(dict)
for (section, key), (
(deprecated_section, deprecated_key, since_version)
) in AirflowConfigParser.deprecated_options.items():
deprecated_options[deprecated_section][deprecated_key] = section, key, since_version
configs = default_config_yaml()
keys_to_format = ["default", "example"]
for conf_name, conf_section in configs.items():
for option_name, option in conf_section["options"].items():
for key in keys_to_format:
if option[key] and "{{" in option[key]:
option[key] = option[key].replace("{{", "{").replace("}}", "}")
for section_name, config in configs.items():
config["options"] = {k: v for k, v in sorted(config["options"].items())}
configs = {k: v for k, v in sorted(configs.items())}
for section in deprecated_options:
deprecated_options[section] = {k: v for k, v in sorted(deprecated_options[section].items())}
jinja_contexts = {
"config_ctx": {"configs": configs, "deprecated_options": deprecated_options},
"quick_start_ctx": {
"doc_root_url": f"https://airflow.apache.org/docs/apache-airflow/{PACKAGE_VERSION}/"
if FOR_PRODUCTION
else (
"http://apache-airflow-docs.s3-website.eu-central-1.amazonaws.com/docs/apache-airflow/latest/"
)
},
"official_download_page": {
"base_url": f"https://downloads.apache.org/airflow/{PACKAGE_VERSION}",
"closer_lua_url": f"https://www.apache.org/dyn/closer.lua/airflow/{PACKAGE_VERSION}",
"airflow_version": PACKAGE_VERSION,
},
}
elif PACKAGE_NAME.startswith("apache-airflow-providers-"):
def _load_config():
file_path = PACKAGE_DIR / "config_templates" / "config.yml"
if not file_path.exists():
return {}
with file_path.open() as f:
return yaml.safe_load(f)
config = _load_config()
jinja_contexts = {
"config_ctx": {"configs": config},
"official_download_page": {
"base_url": "https://downloads.apache.org/airflow/providers",
"closer_lua_url": "https://www.apache.org/dyn/closer.lua/airflow/providers",
"package_name": PACKAGE_NAME,
"package_name_underscores": PACKAGE_NAME.replace("-", "_"),
"package_version": PACKAGE_VERSION,
},
}
elif PACKAGE_NAME == "apache-airflow-providers":
jinja_contexts = {
"official_download_page": {
"all_providers": ALL_PROVIDER_YAMLS,
},
}
elif PACKAGE_NAME == "helm-chart":
def _str_representer(dumper, data):
style = "|" if "\n" in data else None
return dumper.represent_scalar("tag:yaml.org,2002:str", data, style)
yaml.add_representer(str, _str_representer)
def _format_default(value: Any) -> str:
if value == "":
return '""'
if value is None:
return "~"
return str(value)
def _format_examples(param_name: str, schema: dict) -> str | None:
if not schema.get("examples"):
return None
out = ""
for ex in schema["examples"]:
if schema["type"] == "array":
ex = [ex]
out += yaml.dump({param_name: ex})
return out
def _get_params(root_schema: dict, prefix: str = "", default_section: str = "") -> list[dict]:
"""
Given an jsonschema objects properties dict, return a flattened list of all parameters
from that object and any nested objects
"""
out = []
for param_name, schema in root_schema.items():
prefixed_name = f"{prefix}.{param_name}" if prefix else param_name
section_name = schema["x-docsSection"] if "x-docsSection" in schema else default_section
if section_name and schema["description"] and "default" in schema:
out.append(
{
"section": section_name,
"name": prefixed_name,
"description": schema["description"],
"default": _format_default(schema["default"]),
"examples": _format_examples(param_name, schema),
}
)
if schema.get("properties"):
out += _get_params(schema["properties"], prefixed_name, section_name)
return out
schema_file = PACKAGE_DIR / "values.schema.json"
with schema_file.open() as config_file:
chart_schema = json.load(config_file)
params = _get_params(chart_schema["properties"])
sections: dict[str, list[dict[str, str]]] = {}
for param in params:
if param["section"] not in sections:
sections[param["section"]] = []
sections[param["section"]].append(param)
for section in sections.values():
section.sort(key=lambda i: i["name"])
ordered_sections = []
for name in chart_schema["x-docsSectionOrder"]:
if name not in sections:
raise ValueError(f"Unable to find any parameters for section: {name}")
ordered_sections.append({"name": name, "params": sections.pop(name)})
if sections:
raise ValueError(f"Found section(s) which were not in `section_order`: {list(sections.keys())}")
jinja_contexts = {
"params_ctx": {"sections": ordered_sections},
"official_download_page": {
"base_url": "https://downloads.apache.org/airflow/helm-chart",
"closer_lua_url": "https://www.apache.org/dyn/closer.lua/airflow/helm-chart",
"package_name": PACKAGE_NAME,
"package_version": PACKAGE_VERSION,
},
}
autodoc_mock_imports = [
"MySQLdb",
"adal",
"analytics",
"azure",
"azure.cosmos",
"azure.datalake",
"azure.kusto",
"azure.mgmt",
"boto3",
"botocore",
"bson",
"cassandra",
"celery",
"cloudant",
"cryptography",
"datadog",
"distributed",
"docker",
"google",
"google_auth_httplib2",
"googleapiclient",
"grpc",
"hdfs",
"httplib2",
"jaydebeapi",
"jenkins",
"jira",
"kubernetes",
"msrestazure",
"oss2",
"oracledb",
"pandas",
"pandas_gbq",
"paramiko",
"pinotdb",
"psycopg2",
"pydruid",
"pyhive",
"pyhive",
"pymongo",
"pymssql",
"pysftp",
"qds_sdk",
"redis",
"simple_salesforce",
"slack_sdk",
"smbclient",
"snowflake",
"sqlalchemy-drill",
"sshtunnel",
"telegram",
"tenacity",
"vertica_python",
"winrm",
"zenpy",
]
autodoc_default_options = {"show-inheritance": True, "members": True}
autodoc_typehints = "description"
autodoc_typehints_description_target = "documented"
autodoc_typehints_format = "short"
intersphinx_mapping = {
pkg_name: (f"{THIRD_PARTY_INDEXES[pkg_name]}/", (f"{INVENTORY_CACHE_DIR}/{pkg_name}/objects.inv",))
for pkg_name in [
"boto3",
"celery",
"docker",
"hdfs",
"jinja2",
"mongodb",
"pandas",
"python",
"requests",
"sqlalchemy",
]
}
if PACKAGE_NAME in ("apache-airflow-providers-google", "apache-airflow"):
intersphinx_mapping.update(
{
pkg_name: (
f"{THIRD_PARTY_INDEXES[pkg_name]}/",
(f"{INVENTORY_CACHE_DIR}/{pkg_name}/objects.inv",),
)
for pkg_name in [
"google-api-core",
"google-cloud-automl",
"google-cloud-bigquery",
"google-cloud-bigquery-datatransfer",
"google-cloud-bigquery-storage",
"google-cloud-bigtable",
"google-cloud-container",
"google-cloud-core",
"google-cloud-datacatalog",
"google-cloud-datastore",
"google-cloud-dlp",
"google-cloud-kms",
"google-cloud-language",
"google-cloud-monitoring",
"google-cloud-pubsub",
"google-cloud-redis",
"google-cloud-spanner",
"google-cloud-speech",
"google-cloud-storage",
"google-cloud-tasks",
"google-cloud-texttospeech",
"google-cloud-translate",
"google-cloud-videointelligence",
"google-cloud-vision",
]
}
)
viewcode_follow_imported_members = True
autoapi_dirs: list[os.PathLike] = []
if PACKAGE_NAME != "docker-stack":
autoapi_dirs.append(PACKAGE_DIR)
if SYSTEM_TESTS_DIR and os.path.exists(SYSTEM_TESTS_DIR):
autoapi_dirs.append(SYSTEM_TESTS_DIR)
if PACKAGE_NAME == "apache-airflow":
autoapi_template_dir = "autoapi_templates"
autoapi_ignore = [
"*/airflow/_vendor/*",
"*/airflow/executors/*",
"*/_internal*",
"*/node_modules/*",
"*/migrations/*",
"*/contrib/*",
"**/example_sla_dag.py",
"**/example_taskflow_api_docker_virtualenv.py",
"**/example_dag_decorator.py",
]
if PACKAGE_NAME == "apache-airflow":
autoapi_ignore.append("*/airflow/providers/*")
elif PACKAGE_NAME == "docker-stack":
autoapi_ignore.append("*/airflow/providers/*")
else:
autoapi_ignore.append("*/airflow/providers/cncf/kubernetes/backcompat/*")
autoapi_ignore.append("*/example_dags/*")
autoapi_keep_files = True
autoapi_root = "_api"
autoapi_add_toctree_entry = False
autoapi_options = [
"members",
"undoc-members",
"show-inheritance",
"show-module-summary",
"special-members",
]
suppress_warnings = [
"autoapi.python_import_resolution",
]
exampleinclude_sourceroot = os.path.abspath("..")
redirects_file = "redirects.txt"
spelling_word_list_filename = [os.path.join(CONF_DIR, "spelling_wordlist.txt")]
if PACKAGE_NAME == "apache-airflow":
spelling_exclude_patterns = ["project.rst", "changelog.rst"]
if PACKAGE_NAME == "helm-chart":
spelling_exclude_patterns = ["changelog.rst"]
spelling_ignore_contributor_names = False
spelling_ignore_importable_modules = True
graphviz_output_format = "svg"
if PACKAGE_NAME == "apache-airflow":
OPENAPI_FILE = os.path.join(
os.path.dirname(__file__), "..", "airflow", "api_connexion", "openapi", "v1.yaml"
)
redoc = [
{
"name": "Airflow REST API",
"page": "stable-rest-api-ref",
"spec": OPENAPI_FILE,
"opts": {
"hide-hostname": True,
"no-auto-auth": True,
},
},
]
redoc_script_url = "https://cdn.jsdelivr.net/npm/redoc@2.0.0-rc.48/bundles/redoc.standalone.js"
def skip_util_classes(app, what, name, obj, skip, options):
if (what == "data" and "STATICA_HACK" in name) or ":sphinx-autoapi-skip:" in obj.docstring:
skip = True
return skip
def setup(sphinx):
if "autoapi.extension" in extensions:
sphinx.connect("autoapi-skip-member", skip_util_classes)