The Metadata Platform for the Modern Data Stack
Last updated
python3 -m pip install --upgrade pip wheel setuptools
python3 -m pip install --upgrade acryl-datahub
datahub versionDataHub CLI version: 0.10.1
Python version: 3.9.15 | packaged by conda-forge | (main, Nov 22 2022, 08:52:10)
[Clang 14.0.6 ]datahub docker quickstart
datahub docker quickstart --arch m1 # Apple silicon issue# ~/.datahub
networks:
default:
name: datahub_network
services:
broker:
container_name: broker
depends_on:
- zookeeper
environment:
- KAFKA_BROKER_ID=1
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
- KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
- KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
- KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
- KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
- KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false
hostname: broker
image: confluentinc/cp-kafka:7.2.2
ports:
- ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092
datahub-actions:
depends_on:
- datahub-gms
environment:
- DATAHUB_GMS_HOST=datahub-gms
- DATAHUB_GMS_PORT=8080
- DATAHUB_GMS_PROTOCOL=http
- DATAHUB_SYSTEM_CLIENT_ID=__datahub_syste
...
...
...# ~/.datahub/quickstart/docker-compose.yml 을 변경하거나 복사본을 만들어 수정
datahub-frontend-react:
container_name: datahub-frontend-react
depends_on:
- datahub-gms
environment:
- DATAHUB_GMS_HOST=datahub-gms
- DATAHUB_GMS_PORT=8080
- DATAHUB_SECRET=YouKnowNothing
- DATAHUB_APP_VERSION=1.0
- DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB
- JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf
-Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml
-Dlogback.debug=false -Dpidfile.path=/dev/null
- KAFKA_BOOTSTRAP_SERVER=broker:29092
- DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1
- ELASTIC_CLIENT_HOST=elasticsearch
- ELASTIC_CLIENT_PORT=9200
hostname: datahub-frontend-react
image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head}
ports:
# - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 # before change
- ${DATAHUB_MAPPED_FRONTEND_PORT:-8895}:9002 # after change
volumes:
- ${HOME}/.datahub/plugins:/etc/datahub/pluginsdatahub docker quickstart --stopdatahub docker quickstart -f ~/.datahub/quickstart/docker-compose-custom.yml(base) ➜ ~ docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
c5242ec43ac8 linkedin/datahub-frontend-react:v0.10.0 "/bin/sh -c ./start.…" 18 minutes ago Up 18 minutes (healthy) 0.0.0.0:8895->9002/tcp datahub-frontend-react
1e22b7a66002 acryldata/datahub-actions:head "/bin/sh -c 'dockeri…" 38 minutes ago Up 18 minutes datahub-datahub-actions-1
13f11a194bd9 confluentinc/cp-schema-registry:7.2.2 "/etc/confluent/dock…" 57 minutes ago Up 18 minutes 0.0.0.0:8081->8081/tcp schema-registry
dd9637420312 confluentinc/cp-kafka:7.2.2 "/etc/confluent/dock…" 57 minutes ago Up 18 minutes 0.0.0.0:9092->9092/tcp broker
920d1ab8147e linkedin/datahub-gms:v0.10.0 "/bin/sh -c /datahub…" 57 minutes ago Up 18 minutes (healthy) 0.0.0.0:8080->8080/tcp datahub-gms
c6935472793a elasticsearch:7.10.1 "/tini -- /usr/local…" 57 minutes ago Up 18 minutes (healthy) 0.0.0.0:9200->9200/tcp, 9300/tcp elasticsearch
6f6e387edeb5 confluentinc/cp-zookeeper:7.2.2 "/etc/confluent/dock…" 57 minutes ago Up 18 minutes 2888/tcp, 0.0.0.0:2181->2181/tcp, 3888/tcp zookeeper
ada557edf61d mariadb:10.5.8 "docker-entrypoint.s…" 57 minutes ago Up 18 minutes 0.0.0.0:3306->3306/tcp mysql
462c72836930 kindest/node:v1.25.3 "/usr/local/bin/entr…" 2 hours ago Up 2 hours 127.0.0.1:54067->6443/tcp kind-control-plane
(base) ➜ ~
datahub docker quickstart --mysql-port 53306(base) ➜ quickstart datahub docker quickstart --help
Usage: datahub docker quickstart [OPTIONS]
Start an instance of DataHub locally using docker-compose.
This command will automatically download the latest docker-compose configuration from GitHub, pull the latest
images, and bring up the DataHub system. There are options to override the docker-compose config file, build the
containers locally, and dump logs to the console or to a file if something goes wrong.
Options:
--version TEXT Datahub version to be deployed. If not set, deploy using the defaults from the
quickstart compose. Use 'stable' to start the latest stable version.
--build-locally Attempt to build the containers locally before starting
--pull-images / --no-pull-images
Attempt to pull the containers from Docker Hub before starting
-f, --quickstart-compose-file FILE
Use a local docker-compose file instead of pulling from GitHub
--dump-logs-on-failure If true, the docker-compose logs will be printed to console if something fails
--graph-service-impl TEXT If set, forces docker-compose to use that graph service implementation
--mysql-port POSITIVEINT If there is an existing mysql instance running on port 3306, set this to a free port
to avoid port conflicts on startup
--zk-port POSITIVEINT If there is an existing zookeeper instance running on port 2181, set this to a free
port to avoid port conflicts on startup
--kafka-broker-port POSITIVEINT
If there is an existing Kafka broker running on port 9092, set this to a free port
to avoid port conflicts on startup
--schema-registry-port POSITIVEINT
If there is an existing process running on port 8081, set this to a free port to
avoid port conflicts with Kafka schema registry on startup
--elastic-port POSITIVEINT If there is an existing Elasticsearch instance running on port 9092, set this to a
free port to avoid port conflicts on startup
--stop Use this flag to stop the running containers
--backup Run this to backup a running quickstart instance
--backup-file FILE Run this to backup data from a running quickstart instance [default:
/Users/ghlee/.datahub/quickstart/backup.sql]
--restore Run this to restore a running quickstart instance from a previous backup (see
--backup)
--restore-file TEXT Set this to provide a custom restore file
--restore-indices Enable the restoration of indices of a running quickstart instance. Note: Using
--restore will automatically restore-indices unless you use the --no-restore-indices
flag.
--no-restore-indices Disables the restoration of indices of a running quickstart instance when used in
conjunction with --restore.
--standalone_consumers Launches MAE & MCE consumers as stand alone docker containers
--kafka-setup Launches Kafka setup job as part of the compose deployment
--arch TEXT Specify the architecture for the quickstart images to use. Options are x86, arm64,
m1 etc.
--help Show this message and exit.
datahub docker quickstart --stopdatahub docker quickstart --quickstart-compose-file <path to compose file>