Use secret environment variables

Astro environment variables marked as secrets are stored in a Kubernetes secret called env-secrets. To use a secret value in a task running on the Kubernetes executor, you pull the value from env-secrets and mount it to the Pod running your task as a new Kubernetes Secret.

Setup

1

Add import to your dag file

Add the Secret import to your dag file:

1 from airflow.kubernetes.secret import Secret

2

Define a Kubernetes secret

Define a Kubernetes Secret in your dag instantiation using the following format:

1 secret_env = Secret(deploy_type="env", deploy_target="<VARIABLE_KEY>", secret="env-secrets", key="<VARIABLE_KEY>")
2 namespace = conf.get("kubernetes", "NAMESPACE")

3

Reference the environment variable key

Reference the key for the environment variable, formatted as $VARIABLE_KEY in the task using the KubernetesPodOperator.

Example

In the following example, a secret named MY_SECRET is pulled from env-secrets and printed to logs.

1 import pendulum
2 from airflow.kubernetes.secret import Secret
3 
4 from airflow.models import DAG
5 from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator
6 from airflow.configuration import conf
7 
8 with dag(
9         dag_id='test-kube-pod-secret',
10         start_date=pendulum.datetime(2022, 1, 1, tz="UTC"),
11         end_date=pendulum.datetime(2022, 1, 5, tz="UTC"),
12         schedule_interval="@once",
13 ) as dag:
14 
15     secret_env = Secret(deploy_type="env", deploy_target="MY_SECRET", secret="env-secrets", key="MY_SECRET")
16 
17     namespace = conf.get("kubernetes", "NAMESPACE")
18 
19     k = KubernetesPodOperator(
20         namespace=namespace,
21         image="ubuntu:16.04",
22         cmds=["bash", "-cx"],
23         arguments=["echo $MY_SECRET && sleep 150"],
24         name="test-name",
25         task_id="test-task",
26         get_logs=True,
27         in_cluster=True,
28         secrets=[secret_env],
29     )

1	secret_env = Secret(deploy_type="env", deploy_target="<VARIABLE_KEY>", secret="env-secrets", key="<VARIABLE_KEY>")
2	namespace = conf.get("kubernetes", "NAMESPACE")

1	import pendulum
2	from airflow.kubernetes.secret import Secret
3
4	from airflow.models import DAG
5	from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator
6	from airflow.configuration import conf
7
8	with dag(
9	dag_id='test-kube-pod-secret',
10	start_date=pendulum.datetime(2022, 1, 1, tz="UTC"),
11	end_date=pendulum.datetime(2022, 1, 5, tz="UTC"),
12	schedule_interval="@once",
13	) as dag:
14
15	secret_env = Secret(deploy_type="env", deploy_target="MY_SECRET", secret="env-secrets", key="MY_SECRET")
16
17	namespace = conf.get("kubernetes", "NAMESPACE")
18
19	k = KubernetesPodOperator(
20	namespace=namespace,
21	image="ubuntu:16.04",
22	cmds=["bash", "-cx"],
23	arguments=["echo $MY_SECRET && sleep 150"],
24	name="test-name",
25	task_id="test-task",
26	get_logs=True,
27	in_cluster=True,
28	secrets=[secret_env],
29	)