From 5229bcc47d6cecd29b94b026d138539d071d4ed4 Mon Sep 17 00:00:00 2001
From: "Begerad, Stefan" <stefan@begerad.de>
Date: Tue, 5 Oct 2021 09:54:45 -0400
Subject: [PATCH] feat(dup-cleaner): initial commit

---
 duplicate-cleaner/.gitignore                  |  5 +
 duplicate-cleaner/pom.xml                     | 91 +++++++++++++++++++
 .../de/begerad/duprm/DuplicateCleaner.java    | 70 ++++++++++++++
 .../src/main/java/de/begerad/duprm/Main.java  | 47 ++++++++++
 .../src/main/resources/log4j2.xml             | 32 +++++++
 .../begerad/duprm/TestDuplicateCleaner.java   |  4 +
 .../src/test/resources/input.txt              |  6 ++
 7 files changed, 255 insertions(+)
 create mode 100644 duplicate-cleaner/.gitignore
 create mode 100644 duplicate-cleaner/pom.xml
 create mode 100644 duplicate-cleaner/src/main/java/de/begerad/duprm/DuplicateCleaner.java
 create mode 100644 duplicate-cleaner/src/main/java/de/begerad/duprm/Main.java
 create mode 100644 duplicate-cleaner/src/main/resources/log4j2.xml
 create mode 100644 duplicate-cleaner/src/test/java/de/begerad/duprm/TestDuplicateCleaner.java
 create mode 100644 duplicate-cleaner/src/test/resources/input.txt

diff --git a/duplicate-cleaner/.gitignore b/duplicate-cleaner/.gitignore
new file mode 100644
index 0000000..d3954f9
--- /dev/null
+++ b/duplicate-cleaner/.gitignore
@@ -0,0 +1,5 @@
+.idea
+*~
+log.txt
+output.txt
+target
\ No newline at end of file
diff --git a/duplicate-cleaner/pom.xml b/duplicate-cleaner/pom.xml
new file mode 100644
index 0000000..324787e
--- /dev/null
+++ b/duplicate-cleaner/pom.xml
@@ -0,0 +1,91 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <name>dup-rm</name>
+    <description>remove duplicate lines from file</description>
+    <url>https://begerad.de</url>
+    <groupId>de.begerad.duprm</groupId>
+    <artifactId>dup-rm</artifactId>
+    <version>0.0.1</version>
+    <packaging>jar</packaging>
+
+    <properties>
+        <!-- https://maven.apache.org/general.html#encoding-warning -->
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <!-- note that Maven Shade Plugin only works with Java from version 7 and up -->
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
+    </properties>
+
+    <dependencies>
+        <!-- junit 5, unit test -->
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-engine</artifactId>
+            <version>5.7.2</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-slf4j-impl</artifactId>
+            <version>2.14.1</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <!--The default maven-surefire-plugin is outdated,
+            make sure update to the latest.-->
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>2.22.2</version>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.2.4</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <!-- tells that it should be run in package phase -->
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <filters>
+                                <filter>
+                                    <artifact>*:*</artifact>
+                                    <excludes>
+                                        <exclude>META-INF/*.MF</exclude>
+                                    </excludes>
+                                </filter>
+                            </filters>
+                            <shadedArtifactAttached>true</shadedArtifactAttached>
+                            <shadedClassifierName>depds</shadedClassifierName>
+                            <transformers>
+                                <!-- configure resources transformer for Maven Shade Plugin with ManifestResourceTransformer that allows us to define a mainClass in the META-INF/MANIFEST.MF file of the jar file, making the jar file after building, can run standalone -->
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <!-- define some other information in the META-INF/MANIFEST.MF file inside the jar file -->
+                                    <manifestEntries>
+                                        <Main-Class>de.begerad.duprm.Main</Main-Class>
+                                        <Specification-Title>${project.artifactId}</Specification-Title>
+                                        <Specification-Version>${project.version}</Specification-Version>
+                                        <Implementation-Title>${project.artifactId}</Implementation-Title>
+                                        <Implementation-Version>${project.version}</Implementation-Version>
+                                        <Implementation-Vendor-Id>${project.groupId}</Implementation-Vendor-Id>
+                                    </manifestEntries>
+                                </transformer>
+                            </transformers>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>
diff --git a/duplicate-cleaner/src/main/java/de/begerad/duprm/DuplicateCleaner.java b/duplicate-cleaner/src/main/java/de/begerad/duprm/DuplicateCleaner.java
new file mode 100644
index 0000000..ad0ffa8
--- /dev/null
+++ b/duplicate-cleaner/src/main/java/de/begerad/duprm/DuplicateCleaner.java
@@ -0,0 +1,70 @@
+package de.begerad.duprm;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Scanner;
+import java.util.Set;
+
+public class DuplicateCleaner {
+
+    public final static Logger LOG = LoggerFactory.getLogger(DuplicateCleaner.class);
+
+    public static void clean(final String pathInput, final String pathOutput) {
+        LOG.debug("clean() started.");
+
+        String line;
+
+        //instantiating the Scanner class
+        Scanner sc;
+        try {
+            sc = new Scanner(new File(pathInput));
+        } catch (FileNotFoundException e) {
+            LOG.error("clean() ERROR while accessing " + pathInput + " file");
+            LOG.error("clean() stack trace: " + e);
+            return;
+        }
+
+        //instantiating the FileWriter class
+        FileWriter writer;
+        try {
+            writer = new FileWriter(pathOutput);
+        } catch (IOException e) {
+            LOG.error("clean() ERROR while accessing " + pathOutput + " file");
+            LOG.error("clean() stack trace: " + e);
+            return;
+        }
+
+        //instantiating the Set class
+        Set<String> set = new HashSet<>();
+
+        while (sc.hasNextLine()) {
+            line = sc.nextLine();
+            if (set.add(line)) {
+                try {
+                    writer.append(line).append(System.getProperty("line.separator"));
+                } catch (IOException e) {
+                    LOG.error("clean() ERROR while appending data");
+                    LOG.error("clean() stack trace: " + e);
+                    return;
+                }
+            }
+        }
+        try {
+            writer.flush();
+        } catch (IOException e) {
+            LOG.error("clean() ERROR while flushing data");
+            LOG.error(e.getMessage());
+            e.printStackTrace();
+            return;
+        }
+
+        LOG.debug("clean() done.");
+    }
+}
diff --git a/duplicate-cleaner/src/main/java/de/begerad/duprm/Main.java b/duplicate-cleaner/src/main/java/de/begerad/duprm/Main.java
new file mode 100644
index 0000000..5cb84f4
--- /dev/null
+++ b/duplicate-cleaner/src/main/java/de/begerad/duprm/Main.java
@@ -0,0 +1,47 @@
+package de.begerad.duprm;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+
+import static de.begerad.duprm.DuplicateCleaner.clean;
+
+public class Main {
+
+    public static String pathInput = "";
+
+    public static String pathOutput = "";
+
+    public final static Logger LOG = LoggerFactory.getLogger(Main.class);
+
+    public static void main(String[] args) {
+        LOG.debug("main() started...");
+
+        //check user input for input path
+        if (args.length < 1) {
+            System.out.println("Please enter input path as first parameter.");
+            return;
+        }
+
+        pathInput = args[0];
+        if (!(new File(pathInput).isFile())) {
+            System.out.println("parameter: "
+                    + pathInput
+                    + " is NOT a valid path.");
+            return;
+        }
+
+        //check user input for output path
+        if (args.length < 2) {
+            System.out.println("Please enter output path as second parameter.");
+            return;
+        }
+
+        pathOutput = args[1];
+        //call duplicate cleaner
+        clean(pathInput, pathOutput);
+
+        LOG.debug("main() done.");
+    }
+}
diff --git a/duplicate-cleaner/src/main/resources/log4j2.xml b/duplicate-cleaner/src/main/resources/log4j2.xml
new file mode 100644
index 0000000..ce0e7ae
--- /dev/null
+++ b/duplicate-cleaner/src/main/resources/log4j2.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="INFO">
+  <Appenders>
+    <RollingFile name="rollingFile"
+		 fileName="log.txt"
+		 filePattern="log-%d{yyyy-MM-dd}.txt"
+		 ignoreExceptions="false"
+		 >
+      <PatternLayout>
+	<Pattern>[%-5p] %d{yyyy-MM-dd HH:mm:ss.SSS} %c{1} %m%n</Pattern>
+      </PatternLayout>
+      <Policies>
+	<SizeBasedTriggeringPolicy size="3MB" />
+      </Policies>
+      <!--ensure that within the same rollover period no more than X files will be created when a size-based rollover was triggered-->
+      <DefaultRolloverStrategy max="5" />
+    </RollingFile>
+
+    <Console name="console" target="SYSTEM_OUT">
+      <PatternLayout   pattern="[%-5level] %d{yyyy-MM-dd HH:mm:ss.SSS} %c{1} - %msg%n" />
+    </Console>
+  </Appenders>
+  <Loggers>
+    <Logger name="de.begerad" level="debug" additivity="true">
+      <appender-ref ref="rollingFile" level="debug" />
+    </Logger>
+    <Root level="debug" additivity="false">
+      <appender-ref ref="console" />
+    </Root>
+  </Loggers>
+
+</Configuration>
diff --git a/duplicate-cleaner/src/test/java/de/begerad/duprm/TestDuplicateCleaner.java b/duplicate-cleaner/src/test/java/de/begerad/duprm/TestDuplicateCleaner.java
new file mode 100644
index 0000000..6fa0860
--- /dev/null
+++ b/duplicate-cleaner/src/test/java/de/begerad/duprm/TestDuplicateCleaner.java
@@ -0,0 +1,4 @@
+package de.begerad.duprm;
+
+public class TestDuplicateCleaner {
+}
diff --git a/duplicate-cleaner/src/test/resources/input.txt b/duplicate-cleaner/src/test/resources/input.txt
new file mode 100644
index 0000000..9eda46a
--- /dev/null
+++ b/duplicate-cleaner/src/test/resources/input.txt
@@ -0,0 +1,6 @@
+Hello how are you
+Hello how are you
+welcome to Tutorialspoint
+Hello how are you
+Hello how are you
+welcome to Tutorialspoint