cd kafka-proj
docker-compose up -d
#ksql
docker-compose exec ksql-cli ksql http://ksql-server:8088
#bash
docker-compose exec kafka bash
>

Topic

#create topic
docker-compose exec broker \
kafka-topics --bootstrap-server broker:29092 \
--create \
--replication-factor 1 \
--partitions 3 \
--topic test;
  • Read/Write
#go in kafka broker
docker-compose exec broker bash
#read topic
kafka-console-consumer --bootstrap-server broker:9092…

All sql files in folder

Execute in docker container
docker exec -i container-name {executing command}

for sql_file in ls {path}/*.sql
do
docker exec -i mysql-container-name mysql -utest_user -p"test-pws" "db_name" -h mysql < $sql_file ;
done

Execute mysql in linux
mysql -utest_user -p”test-pws” “db_name” -h mysql {executing command}

for sql_file in ls {path}/*.sql
do
mysql -utest_user -p"test-pws" "db_name" -h mysql < $sql_file ;
done

Run mysql-initial.sh

cat {path}/mysql-initial.sh | docker exec -i mysql-container-name mysql -uroot -p$MYSQL_ROOT_PASSWORD -h mysql

{path}/mysql-initial.sh

CREATE USER IF NOT EXISTS 'test_user'@'%' IDENTIFIED BY 'test-pws';
CREATE DATABASE IF NOT EXISTS `db_name` DEFAULT CHARACTER SET `utf8` COLLATE `utf8_general_ci`;
GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, CREATE TEMPORARY TABLES, DROP, INDEX, ALTER, LOCK TABLES ON `db_name`.* TO 'test_user'@'%';

Elastic search

deployment.yml

apiVersion: apps/v1
kind: Deployment
metadata:
name: elasticsearch-testing
spec:
replicas: 1
selector:
matchLabels:
component: elasticsearch-testing
template:
metadata:
labels:
component: elasticsearch-testing
spec:
containers:
- name: elasticsearch-testing
image: docker.elastic.co/elasticsearch/elasticsearch:7.14.0
resources:
limits:
memory: 1Gi
env:
- name: discovery.type
value: single-node
ports:
- containerPort: 9200
name: http
protocol: TCP

service.yml

apiVersion: v1
kind: Service
metadata:
name: elasticsearch-testing-service
labels:
component: elasticsearch-testing
spec:
type: LoadBalancer
loadBalancerSourceRanges:
- {ip}/32
selector:
component: elasticsearch-testing
ports:
- name: http
port: 9200
protocol: TCP

Kibana

deployment.yml

apiVersion: apps/v1
kind: Deployment
metadata:
name: userprofile-kibana
spec:
replicas: 1
selector:
matchLabels:
component: una-kibana
template:
metadata:
labels:
component: una-kibana
spec:
containers:
- name: una-kibana
image: docker.elastic.co/kibana/kibana:7.14.0
env:
- name: ELASTICSEARCH_HOSTS
value: http://elasticsearch-testing-service:9200
ports:
- containerPort: 5601
name: http
protocol: TCP

service.yml

apiVersion: v1
kind: Service
metadata:
name: userprofile-kibana-service
labels:
component: una-kibana
spec:
type: LoadBalancer
loadBalancerSourceRanges:
- {ip}/32
selector:
component: userprofile-kibana
ports:
- name: http
port: 5601
protocol: TCP

data_unit: daily/weekly/monthly

start_date: start date of period

end_date: finish date of period

Example

Daily
daily_list = get_period_date_by_unit(data_unit= ‘daily’, start_date= ‘2021–9–1’, end_date= ‘2021–9–5’)
#daily_list=[‘2021–9–1’,‘2021–9–2’,‘2021–9–3’,‘2021–9–4’,‘2021–9–5’]

Weekly
#Sunday(isoweekday=0) is base day
weekly_list = get_period_date_by_unit(data_unit= ‘daily’, start_date= ‘2021–9–1’, end_date= ‘2021–9–30’)
#weekly_list=[‘2021–9–5’,‘2021–9–12’,‘2021–9–19’,‘2021–9–26’]

Monthly
monthly_list = get_period_date_by_unit(data_unit= ‘daily’, start_date= ‘2021–9–1’, end_date= ‘2021–12–30’)
#monthly_list=[‘2021–9–1’,‘2021–10–1’,‘2021–11–1’,‘2021–12–1’]

Code

def get_period_date_by_unit(data_unit: str, start_date: dt.datetime,
end_date: dt.datetime) -> list:
if data_unit == 'weekly':
freq = 'W'
if start_date.isoweekday() != 0:
start_date = start_date + dt.timedelta(
days=(-start_date.isoweekday()))
elif data_unit == 'monthly':
freq = 'M'
end_date = end_date + relativedelta(months=1)
else:
freq = 'D'

date_list = list(pd.date_range(start=start_date, end=end_date, freq=freq))

if data_unit == 'monthly':
tz = dt.timezone(dt.timedelta(hours=TIME_ZONE_OFFSET))
date_list = [dt.datetime(d.year, d.month, 1, tzinfo=tz) for d in
date_list]

return date_list

TIME_ZONE_OFFSET = -4
def
transform_utc_timestamp_to_date_str(dt_timestamp: int) -> str:
utc_time = dt.datetime.utcfromtimestamp(dt_timestamp)
proj_time = utc_time + dt.timedelta(hours=TIME_ZONE_OFFSET)

fill_len = 2
if TIME_ZONE_OFFSET < 0:
fill_len = 3

return proj_time.strftime('%Y-%m-%dT%H:%M:%S') + f'Z{str(TIME_ZONE_OFFSET).zfill(fill_len)}:00'

  • .gitlab-ci.yml
default:image: python:3.8.10-slim-busterbefore_script:- bashstages:- build- test- build-image- deploy-gkebuild:stage: buildscript:- apt-get -y update- apt-get -y upgrade- apt-get -y install --no-install-recommends syslog-ng # Install a new package, without unnecessary recommended packages:- apt-get clean- pip install pipenv
$gcloud container clusters get-credentials cluster-id --zone asia-east1-b --project project-id

Setting file

  • ConfigMap.yml
  • service.yml
  • deployment.yml

ConfigMap.yml

apiVersion: v1
kind: ConfigMap
metadata:
name: dev-data-api-config
namespace: default
data:
api.url: |
STREAMING_API=http://data-api-service

service.yml

apiVersion: v1
kind: Service
metadata:
name: data-api-service
spec:
selector:
app: data-api
ports:
- protocol: TCP
port: 80
targetPort: 8999

development.yml

  • ConfigMap…
import timefrom hypothesis import given, settings, Verbosity, example # pip install hypothesisimport hypothesis.strategies as stimport pytestdef get_remainder(item):time.sleep(1)if item == 0:return 0return 2 % abs(item)# return 2 % item@pytest.mark.parametrize(“item”, [1, 3, 5, 7])def test_pytest_get_remainder(item):assert get_remainder(item) >= 0@settings(deadline=1 * 1000, # 指定允許單個示例花費多長時間# verbosity=Verbosity.verbose, # pytest “file” -v -smax_examples=5 # 控制在測試結束之前需要通過多少個示例)@given(item=st.integers())@example(item=1)@example(item=3)@example(item=5)def test_hypothesis_get_remainder(item):assert get_remainder(item) >= 0

Una CHOU

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store