From db16b139fb1c4fc612e000f75c933bf127e5221e Mon Sep 17 00:00:00 2001 From: tawoe Date: Fri, 6 Dec 2019 09:51:57 +0100 Subject: [PATCH] kafka doc hotfix WIP --- README.kafka.md | 26 ++++++ README.md | 108 ------------------------ obp-api/src/main/scripts/kafka/setup.sh | 6 +- 3 files changed, 29 insertions(+), 111 deletions(-) create mode 100644 README.kafka.md diff --git a/README.kafka.md b/README.kafka.md new file mode 100644 index 000000000..f552fd725 --- /dev/null +++ b/README.kafka.md @@ -0,0 +1,26 @@ +## Kafka Quickstart + + Note that obp with kafka connector will also need a bank backend connected to the kafka that implements the following: https://apiexplorersandbox.openbankproject.com/glossary#Adapter.Kafka.Intro + + Otherwise obp will display anything but adapter errors. + +#####Configuration + +* Edit the OBP-API/obp-api/src/main/resources/props/default.props so that it contains the following lines: + + connector=kafka_vMay2019 + # kafka server location, plaintext for quickstart + kafka.host=localhost:9092 + # next 2 lines for legacy resons + kafka.request_topic=Request + kafka.response_topic=Response + # number of partitions available on kafka. Must match the kafka configuration!!!!!!. + kafka.partitions=1 + # no ssl for quickstart + kafka.use.ssl=false + # start with 1 for the first instance, set to 2 for the second instance etc + api_instance_id=1 + + + + diff --git a/README.md b/README.md index 177b6392b..b5e1a5952 100644 --- a/README.md +++ b/README.md @@ -210,114 +210,6 @@ To populate the OBP database with sandbox data: 6) If successful you should see an empty result `{}` and no error message -## Kafka (optional): - -If Kafka connector is selected in props (connector=kafka), Kafka and Zookeeper have to be installed, as well as OBP-Kafka-Python (which can be either running from command-propmpt or from inside Docker container): - -* Kafka and Zookeeper can be installed using system's default installer or by unpacking the archives (http://apache.mirrors.spacedump.net/kafka/ and http://apache.mirrors.spacedump.net/zookeeper/) - -* OBP-Kafka-Python can be downloaded from https://github.com/OpenBankProject/OBP-Kafka-Python - -## Running with a Zookeeper/Kafka Cluster - -1) NGINX Configuration for Load Balancing - -* Create file /etc/nginx/sites-available/api - -* Configure as follows: - - upstream backend { - least_conn; - server host1:8080; # The name of the server shall be changed as appropriate - server host2:8080; - server host3:8080; - } - - server { - server_name obptest.com www.obptest.com; # The server name should be changed as appropriate - access_log /var/log/nginx/api.access.log; - error_log /var/log/nginx/api.error.log; - location / { - proxy_pass http://backend/; - } - location /obp/v2.1.0/sandbox/data-import { - proxy_pass http://backend/; - } - } - -2) Zookeeper/Kafka Cluster Setup - -* The Zookeeper/Kafka cluster is deployed on 3 nodes. The following configurations need to be done on each of the three nodes - -* Zookeeper configuration - -* Inside the Kafka directory, edit the file conf/zookeeper.properties and include these lines: - - dataDir=/home/user/zookeeper - server.1=host1:2888:3888 # The name of the servers shall be changed as appropriate - server.2=host2:2888:3888 - server.3=host3:2888:3888 - initLimit=5 - syncLimit=2 - -* Create a myid file under dataDir which is /home/user/zookeeper in this example: - - echo “1” > /home/user/zookeeper/myid #Insert unique id’s on each of the machines - -* Start the zookpeer daemons on each of the 3 machines - - bin/zookeeper-server-start.sh config/zookeeper.properties & - -* Kafka Configuration - -* Inside the Kafka directory, edit the file conf/server.properties and include these lines: - - broker.id=1 # The broker.id should be unique for each host - - num.partitions=4 - - zookeeper.connect=host1:2181,host2:2181,host3:2181 - -* Start the kafka broker daemons on all the machines: - - bin/kafka-server-start.sh config/server.properties & - -* Create the topics: - - bin/kafka-topics.sh --create --zookeeper host1:2181,host2:2181,host3:2181 --replication-factor 1 --partitions 1 --topic Request - - bin/kafka-topics.sh --create --zookeeper host1:2181,host2:2181,host3:2181 --replication-factor 1 --partitions 1 --topic Response - -3) OBP-API - -* Configuration - -* Edit the OBP-API/obp-api/src/main/resources/props/default.props so that it contains the following lines. Please note that -kafka.host is used by the producer and kafka.zookeeper_host is used by the consumer. This should be done on each node: - - connector=kafka - # Address to be used by consumer - kafka.zookeeper_host=localhost:2181 - # Address to be used by producer - kafka.host=localhost:9092 - kafka.request_topic=Request - kafka.response_topic=Response - -* Start the server: - - cd OBP-API/obp-api/ - mvn jetty:run - -4) OBP-Kafka-Python - -* Run from the command line: - - cd OBP-Kafka-Python - python server.py - -5) To test the setup, try a request - -http://localhost:8080/obp/v2.0.0/banks ## Production Options. diff --git a/obp-api/src/main/scripts/kafka/setup.sh b/obp-api/src/main/scripts/kafka/setup.sh index a9f4e0f2e..ba7a20196 100755 --- a/obp-api/src/main/scripts/kafka/setup.sh +++ b/obp-api/src/main/scripts/kafka/setup.sh @@ -1,3 +1,3 @@ -wget http://mirror.synyx.de/apache/kafka/1.1.0/kafka_2.12-1.1.0.tgz && \ -tar xzf kafka_2.12-1.1.0.tgz && \ -rm -f kafka_2.12-1.1.0.tgz +wget https://archive.apache.org/dist/kafka/1.1.1/kafka_2.12-1.1.1.tgz && \ +tar xzf kafka_2.12-1.1.1.tgz && \ +rm -f kafka_2.12-1.1.1.tgz