From e763a7611b898cfbeaafabaa1a767bf415575b42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Geoffrey=20=E2=80=9CFrogeye=E2=80=9D=20Preud=27homme?= Date: Wed, 23 Jan 2019 21:54:19 +0100 Subject: [PATCH] Flink bootstrap --- .gitignore | 35 +++ pom.xml | 232 ++++++++++++++++++ .../middleware/projects/flink/BatchJob.java | 66 +++++ .../projects/flink/StreamingJob.java | 64 +++++ src/main/resources/log4j.properties | 23 ++ 5 files changed, 420 insertions(+) create mode 100644 .gitignore create mode 100644 pom.xml create mode 100644 src/main/java/it/polimi/middleware/projects/flink/BatchJob.java create mode 100644 src/main/java/it/polimi/middleware/projects/flink/StreamingJob.java create mode 100644 src/main/resources/log4j.properties diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2edcb90 --- /dev/null +++ b/.gitignore @@ -0,0 +1,35 @@ +# Compiled class file +*.class + +# Log file +*.log + +# BlueJ files +*.ctxt + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.nar +*.ear +*.zip +*.tar.gz +*.rar + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* + +# Maven +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +pom.xml.next +release.properties +dependency-reduced-pom.xml +buildNumber.properties +.mvn/timing.properties +.mvn/wrapper/maven-wrapper.jar diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..1088bfe --- /dev/null +++ b/pom.xml @@ -0,0 +1,232 @@ + + + 4.0.0 + + it.polimi.middleware.projects.flink + project + 1.0-SNAPSHOT + jar + + Flink Quickstart Job + http://www.myorganization.org + + + UTF-8 + 1.7.0 + 1.8 + 2.11 + ${java.version} + ${java.version} + + + + + apache.snapshots + Apache Development Snapshot Repository + https://repository.apache.org/content/repositories/snapshots/ + + false + + + true + + + + + + + + + org.apache.flink + flink-java + ${flink.version} + provided + + + org.apache.flink + flink-streaming-java_${scala.binary.version} + ${flink.version} + provided + + + + + + + + + + org.slf4j + slf4j-log4j12 + 1.7.7 + runtime + + + log4j + log4j + 1.2.17 + runtime + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + ${java.version} + ${java.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.0.0 + + + + package + + shade + + + + + org.apache.flink:force-shading + com.google.code.findbugs:jsr305 + org.slf4j:* + log4j:* + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + it.polimi.middleware.projects.flink.StreamingJob + + + + + + + + + + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + org.apache.maven.plugins + maven-shade-plugin + [3.0.0,) + + shade + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + [3.1,) + + testCompile + compile + + + + + + + + + + + + + + + + + + + + add-dependencies-for-IDEA + + + + idea.version + + + + + + org.apache.flink + flink-java + ${flink.version} + compile + + + org.apache.flink + flink-streaming-java_${scala.binary.version} + ${flink.version} + compile + + + + + + diff --git a/src/main/java/it/polimi/middleware/projects/flink/BatchJob.java b/src/main/java/it/polimi/middleware/projects/flink/BatchJob.java new file mode 100644 index 0000000..606742f --- /dev/null +++ b/src/main/java/it/polimi/middleware/projects/flink/BatchJob.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package it.polimi.middleware.projects.flink; + +import org.apache.flink.api.java.ExecutionEnvironment; + +/** + * Skeleton for a Flink Batch Job. + * + *

For a tutorial how to write a Flink batch application, check the + * tutorials and examples on the Flink Website. + * + *

To package your application into a JAR file for execution, + * change the main class in the POM.xml file to this class (simply search for 'mainClass') + * and run 'mvn clean package' on the command line. + */ +public class BatchJob { + + public static void main(String[] args) throws Exception { + // set up the batch execution environment + final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); + + /* + * Here, you can start creating your execution plan for Flink. + * + * Start with getting some data from the environment, like + * env.readTextFile(textPath); + * + * then, transform the resulting DataSet using operations + * like + * .filter() + * .flatMap() + * .join() + * .coGroup() + * + * and many more. + * Have a look at the programming guide for the Java API: + * + * http://flink.apache.org/docs/latest/apis/batch/index.html + * + * and the examples + * + * http://flink.apache.org/docs/latest/apis/batch/examples.html + * + */ + + // execute program + env.execute("Flink Batch Java API Skeleton"); + } +} diff --git a/src/main/java/it/polimi/middleware/projects/flink/StreamingJob.java b/src/main/java/it/polimi/middleware/projects/flink/StreamingJob.java new file mode 100644 index 0000000..55b6a84 --- /dev/null +++ b/src/main/java/it/polimi/middleware/projects/flink/StreamingJob.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package it.polimi.middleware.projects.flink; + +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; + +/** + * Skeleton for a Flink Streaming Job. + * + *

For a tutorial how to write a Flink streaming application, check the + * tutorials and examples on the Flink Website. + * + *

To package your application into a JAR file for execution, run + * 'mvn clean package' on the command line. + * + *

If you change the name of the main class (with the public static void main(String[] args)) + * method, change the respective entry in the POM.xml file (simply search for 'mainClass'). + */ +public class StreamingJob { + + public static void main(String[] args) throws Exception { + // set up the streaming execution environment + final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + + /* + * Here, you can start creating your execution plan for Flink. + * + * Start with getting some data from the environment, like + * env.readTextFile(textPath); + * + * then, transform the resulting DataStream using operations + * like + * .filter() + * .flatMap() + * .join() + * .coGroup() + * + * and many more. + * Have a look at the programming guide for the Java API: + * + * http://flink.apache.org/docs/latest/apis/streaming/index.html + * + */ + + // execute program + env.execute("Flink Streaming Java API Skeleton"); + } +} diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties new file mode 100644 index 0000000..da32ea0 --- /dev/null +++ b/src/main/resources/log4j.properties @@ -0,0 +1,23 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +log4j.rootLogger=INFO, console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n