{"type":"doc","content":[{"type":"paragraph","content":[{"text":"Spark applications must be configured with a set of Kafka topics that are either shared between multiple applications or dedicated to specific applications. The assigned topics must be created before you submit an application to the Spark cluster. Before you can create the topics you must start Zookeeper and Kafka. ","type":"text"}]},{"type":"heading","attrs":{"level":1},"content":[{"text":"Prerequisites:","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Prepare scripts according to ","type":"text"},{"type":"inlineCard","attrs":{"url":"https://infozone.atlassian.net/wiki/spaces/MD82/pages/102924470"}},{"text":" ","type":"text"}]}]}]},{"type":"heading","attrs":{"level":1},"content":[{"text":"Starting Clusters","type":"text"}]},{"type":"paragraph","content":[{"text":"To start a cluster follow the steps:","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Start Zookeeper and Kafka ","type":"text"},{"type":"hardBreak"},{"text":"To start Zookeeper, run the following:","type":"text"}]},{"type":"paragraph","content":[{"text":"bin/zookeeper-server-start.sh config/zookeeper.properties","type":"text","marks":[{"type":"code"}]},{"type":"hardBreak"},{"type":"hardBreak"},{"text":"To start Kafka, run:","type":"text"},{"type":"hardBreak"},{"text":"bin/kafka-server-start.sh config/server.properties","type":"text","marks":[{"type":"code"}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Create Kafka topics and partitions using the scripts included in the Kafka installation. ","type":"text"},{"type":"hardBreak"},{"text":"The names of the topics must correspond to the Spark application configuration.","type":"text"},{"type":"hardBreak"}]}]}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"In order for the Spark KPI Application to work, the required number of partitions for each topic must be equal to the setting of the property ","type":"text"},{"text":"spark.default.parallelism","type":"text","marks":[{"type":"code"}]},{"text":" in the Spark application configuration.","type":"text"}]}]},{"type":"paragraph","content":[{"text":"Use a replication factor that is greater than one (1) to make sure that data is replicated between Kafka brokers. This decreases the risk of losing data in case of issues with the brokers. This is how to create topics, assuming the current working directory is the Kafka software folder:","type":"text"}]},{"type":"codeBlock","attrs":{"language":"plaintext"},"content":[{"text":"$ ./bin/kafka-topics.sh --create --topic --bootstrap-server \\\nlocalhost:9092 --partitions --replication-factor \n$ ./bin/kafka-topics.sh --create --topic --bootstrap-server \\\nlocalhost:9092 --partitions --replication-factor \n$ ./bin/kafka-topics.sh --create --topic --bootstrap-server \\\nlocalhost:9092 --partitions --replication-factor \n","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Example - Creating Kafka Topics","type":"text","marks":[{"type":"strong"}]}]},{"type":"codeBlock","content":[{"text":"./bin/kafka-topics.sh --create --topic kpi-output --partitions 6 --replication-factor 1\n./bin/kafka-topics.sh --create --topic kpi-input --partitions 6 --replication-factor 1\n./bin/kafka-topics.sh --create --topic kpi-alarm --partitions 6 --replication-factor 1","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Example - Creating Kafka topics, overriding retention settings","type":"text"}]},{"type":"codeBlock","content":[{"text":"./bin/kafka-topics.sh --create --topic kpi-output --partitions 6 --replication-factor 1 --config retention.ms=86400000\n./bin/kafka-topics.sh --create --topic kpi-input --partitions 6 --replication-factor 1 --config retention.ms=86400000\n./bin/kafka-topics.sh --create --topic kpi-alarm --partitions 6 --replication-factor 1 --config retention.ms=86400000","type":"text"}]},{"type":"paragraph"},{"type":"orderedList","attrs":{"order":3},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Run the following command to start Spark:","type":"text"},{"type":"hardBreak"},{"text":"$ start_master_workers.sh ...","type":"text","marks":[{"type":"code"}]},{"type":"hardBreak"},{"text":"To submit the app to the Spark cluster","type":"text"}]},{"type":"paragraph","content":[{"text":". Submit the app:","type":"text","marks":[{"type":"code"}]},{"type":"hardBreak"},{"text":" $ submit.sh kpiapp ...","type":"text","marks":[{"type":"code"}]},{"type":"hardBreak"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"You can now confirm the status of the Spark cluster. Open a browser and go to ","type":"text"},{"text":"http://:8080","type":"text","marks":[{"type":"code"}]},{"text":".","type":"text"},{"type":"hardBreak"},{"text":" ","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"mediaSingle","attrs":{"layout":"center","width":760,"widthType":"pixel"},"content":[{"type":"media","attrs":{"width":938,"id":"f1c8f535-57c5-4dc7-94d8-960f713e217e","collection":"contentId-102924667","type":"file","height":704}}]},{"type":"paragraph","content":[{"text":" ","type":"text","marks":[{"type":"strong"}]},{"text":" ","type":"text"},{"type":"hardBreak"},{"text":" Spark UI ","type":"text","marks":[{"type":"strong"}]}]},{"type":"bodiedExtension","attrs":{"layout":"default","extensionType":"com.atlassian.confluence.macro.core","extensionKey":"scroll-ignore","parameters":{"macroParams":{"scroll-viewport":{"value":"false"},"scroll-pdf":{"value":"true"},"scroll-office":{"value":"false"},"scroll-chm":{"value":"true"},"scroll-html":{"value":"false"},"scroll-docbook":{"value":"true"},"scroll-eclipsehelp":{"value":"true"},"scroll-epub":{"value":"true"}},"macroMetadata":{"macroId":{"value":"3f04d6bc-7e9a-4632-8809-95870ff82c75"},"schemaVersion":{"value":"1"},"placeholder":[{"type":"icon","data":{"url":"https://scroll-exporter-extensions.us.exporter.k15t.app/static/assets/images/macro-icons/scroll-ignore.svg"}}],"title":"Scroll Ignore"}},"localId":"83619b54-a99e-4227-a88f-579f669b561e"},"content":[{"type":"paragraph","content":[{"text":"4.3.4 Using Multiple Service Models","type":"text","marks":[{"type":"link","attrs":{"href":"/wiki/spaces/MD82/pages/102924703"}}]}]}]}],"version":1}

Browser not supported