From c5d8892d8921ceecc59300b9d06a18e2507d0d85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Doursenaud?= <rdoursenaud@gpcsolutions.fr>
Date: Wed, 26 Mar 2014 18:12:48 +0100
Subject: [PATCH] Added a dev script to recursively deduplicate file lines on a
 per file basis

This script is useful to deduplicate language files
---
 dev/deduplicatefilelinesrecursively.sh | 34 ++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100755 dev/deduplicatefilelinesrecursively.sh

diff --git a/dev/deduplicatefilelinesrecursively.sh b/dev/deduplicatefilelinesrecursively.sh
new file mode 100755
index 00000000000..2aaa05c0934
--- /dev/null
+++ b/dev/deduplicatefilelinesrecursively.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+# Recursively deduplicate file lines on a per file basis
+# Useful to deduplicate language files
+#
+# Needs awk 4.0 for the inplace fixing command
+#
+# Raphaël Doursenaud - rdoursenaud@gpcsolutions.fr
+
+# Syntax
+if [ "x$1" != "xlist" -a "x$1" != "xfix" ]
+then
+	echo "Usage: deduplicatefilelinesrecursively.sh [list|fix]"
+fi
+
+# To detect
+if [ "x$1" = "xlist" ]
+then
+    for file in `find . -type f`
+    do
+        if [ `sort "$file" | uniq -d | wc -l` -gt 0 ]
+        then
+            echo "$file"
+        fi
+    done
+fi
+
+# To fix
+if [ "x$1" = "xfix" ]
+then
+    for file in `find . -type f`
+    do
+    	awk -i inplace ' !x[$0]++' "$file"
+    done;
+fi