single node airflow for batching data ingestion and processing
airflow는 일단 로컬에서 설치형으로 작업하되, 외부 필요한 서비스들은 docker로 작동
conda create -n air python=3.10
conda activate air
conda env remove -n air # remove env if needed
sh airflow_setup.sh
sh set_env.sh
# airflow.cfg를 입맛에 맞게 수정 한 후 확인
airflow info # check airflow configuration
airflow db init
airflow db upgrade # create the database schema that Airflow can use
airflow db check # db reachable?
airflow db reset # reset db
airflow users create --username admin \
--firstname admin --lastname admin \
--role Admin \
--email darrenkwondev46@gmail.com
airflow users list
airflow webserver --port 8080
airflow scheduler
# task 추가시. 의존 task를 다 실행하지 않고 Test a task instance.
airflow tasks test $dag_id $tasks_id $execution_date(YYYY-MM-DD)
export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES