Fixes #761, #3093 (svn history on renamed files, svn date parsing) (#3095)

Author: harrisric

.gitattributes

@@ -12,3 +12,5 @@
 *.PDF	 diff=astextplain
 *.rtf	 diff=astextplain
 *.RTF	 diff=astextplain
+
+/testdata/svndump/svnlog.dump text eol=lf

opengrok-indexer/build.xml

@@ -30,6 +30,7 @@ Copyright (c) 2007, 2017, Oracle and/or its affiliates. All rights reserved.
 
     <property name="test.repositories" value="../testdata/repositories"/>
     <property name="test.sources" value="../testdata/sources"/>
+    <property name="test.svndump" value="../testdata/svndump"/>
     <property name="test.plugins" value="../testdata/plugins"/>
     <property name="test.cvs" value="${test.repositories}/cvs_test"/>
     <property name="test.cvs.repo" value="${test.cvs}/cvsrepo"/>
@@ -39,6 +40,7 @@ Copyright (c) 2007, 2017, Oracle and/or its affiliates. All rights reserved.
     <property name="test.hg" value="${test.repositories}/mercurial"/>
     <property name="test.git" value="${test.repositories}/git"/>
     <property name="test.svn" value="${test.repositories}/svn"/>
+    <property name="test.svn2" value="${test.repositories}/subversion"/>
     <property name="test.razor" value="${test.repositories}/razor"/>
     <property name="test.razor.repo" value="../ext/SampleRazorRepository/"/>
     <property name="build.test.reposroots" value="target/reposroot"/>
@@ -122,6 +124,10 @@ Copyright (c) 2007, 2017, Oracle and/or its affiliates. All rights reserved.
         <map from="" to="file:///"/>
         <path location="${build.test.reposroots}/svn"/>
       </pathconvert>
+      <pathconvert property="test.svn.url2">
+        <map from="" to="file:///"/>
+        <path location="${build.test.reposroots}/svn2"/>
+      </pathconvert>
 
       <exec executable="svn" failifexecutionfails="false">
         <arg value="import"/>
@@ -135,6 +141,22 @@ Copyright (c) 2007, 2017, Oracle and/or its affiliates. All rights reserved.
         <arg value="${test.svn.url}"/>
         <arg value="${test.svn}"/>
       </exec>
+
+      <exec executable="svnadmin" failifexecutionfails="true">
+        <arg value="create"/>
+        <arg value="${build.test.reposroots}/svn2"/>
+      </exec>
+    	
+      <exec executable="svnadmin" input="${test.svndump}/svnlog.dump" failifexecutionfails="true">
+        <arg value="load"/>
+        <arg value="${build.test.reposroots}/svn2"/>
+      </exec>
+    
+      <exec executable="svn" failifexecutionfails="false">
+        <arg value="checkout"/>
+        <arg value="${test.svn.url2}"/>
+        <arg value="${test.svn2}"/>
+      </exec>
     </target>
 
     <target name="-create-razor-repository">
@@ -158,13 +180,14 @@ Copyright (c) 2007, 2017, Oracle and/or its affiliates. All rights reserved.
     </target>
 
     <!-- clean up generated test repositories -->
-    <target name="-delete-generated-repository-files">
+    <target name="-delete-generated-repository-files"> 
       <delete dir="${test.bk}/.bk"/>
       <delete dir="${test.bzr}/.bzr"/>
       <delete dir="${test.hg}/.hg"/>
       <delete file="${test.hg}/.hgignore"/>
       <delete dir="${test.git}/.git"/>
       <delete dir="${test.svn}"/>
+      <delete dir="${test.svn2}"/>
       <delete dir="${test.razor}"/>
       <delete file="${test.cvs.repo}/CVS/Root"/>
     </target>

opengrok-indexer/src/main/java/org/opengrok/indexer/history/SubversionHistoryParser.java

@@ -20,6 +20,7 @@
 /*
  * Copyright (c) 2006, 2018, Oracle and/or its affiliates. All rights reserved.
  * Portions Copyright (c) 2017, Chris Fraire <cfraire@me.com>.
+ * Portions Copyright (c) 2020, Ric Harris <harrisric@users.noreply.github.com>.
  */
 package org.opengrok.indexer.history;
 
@@ -30,11 +31,15 @@
 import java.io.InputStream;
 import java.text.ParseException;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+
 import javax.xml.parsers.SAXParser;
 import javax.xml.parsers.SAXParserFactory;
+
 import org.opengrok.indexer.configuration.RuntimeEnvironment;
 import org.opengrok.indexer.logger.LoggerFactory;
 import org.opengrok.indexer.util.Executor;
@@ -56,13 +61,20 @@ class SubversionHistoryParser implements Executor.StreamHandler {
 
     private static class Handler extends DefaultHandler2 {
 
+        /**
+         * Example of the longest date format that we should accept - SimpleDateFormat cannot cope with micro/nano seconds.
+         */
+        static final int SVN_MILLIS_DATE_LENGTH = "2020-03-26T15:38:55.999Z".length();
+
         final String prefix;
         final String home;
         final int length;
-        final List<HistoryEntry> entries = new ArrayList<HistoryEntry>();
+        final List<HistoryEntry> entries = new ArrayList<>();
+        final Set<String> renamedFiles = new HashSet<>();
         final SubversionRepository repository;
         HistoryEntry entry;
         StringBuilder sb;
+        boolean isRenamed;
 
         Handler(String home, String prefix, int length, SubversionRepository repository) {
             this.home = home;
@@ -72,12 +84,19 @@ private static class Handler extends DefaultHandler2 {
             sb = new StringBuilder();
         }
 
+        List<String> getRenamedFiles() {
+            return new ArrayList<>(renamedFiles);
+        }
+
         @Override
         public void startElement(String uri, String localName, String qname, Attributes attr) {
+            isRenamed = false;
             if ("logentry".equals(qname)) {
                 entry = new HistoryEntry();
                 entry.setActive(true);
                 entry.setRevision(attr.getValue("revision"));
+            } else if ("path".equals(qname)) {
+                isRenamed = attr.getIndex("copyfrom-path") != -1;
             }
             sb.setLength(0);
         }
@@ -89,7 +108,13 @@ public void endElement(String uri, String localName, String qname) throws SAXExc
                 entry.setAuthor(s);
             } else if ("date".equals(qname)) {
                 try {
-                    entry.setDate(repository.parse(s));
+                    // need to strip microseconds off - assume final character is Z otherwise invalid anyway.
+                    String dateString = s;
+                    if (s.length() > SVN_MILLIS_DATE_LENGTH) {
+                      dateString = dateString.substring(0, SVN_MILLIS_DATE_LENGTH - 1) +
+                          dateString.charAt(dateString.length() - 1);
+                    }
+                    entry.setDate(repository.parse(dateString));
                 } catch (ParseException ex) {
                     throw new SAXException("Failed to parse date: " + s, ex);
                 }
@@ -104,6 +129,9 @@ public void endElement(String uri, String localName, String qname) throws SAXExc
                     // The same file names may be repeated in many commits,
                     // so intern them to reduce the memory footprint.
                     entry.addFile(path.intern());
+                    if (isRenamed) {
+                        renamedFiles.add(file.getAbsolutePath().substring(home.length() + 1));
+                    }
                 } else {
                     LOGGER.log(Level.FINER, "Skipping file outside repository: " + s);
                 }
@@ -177,7 +205,7 @@ History parse(File file, SubversionRepository repos, String sinceRevision,
             repos.removeAndVerifyOldestChangeset(entries, sinceRevision);
         }
 
-        return new History(entries);
+        return new History(entries, handler.getRenamedFiles());
     }
 
    /**
@@ -206,6 +234,6 @@ public void processStream(InputStream input) throws IOException {
     History parse(String buffer) throws IOException {
         handler = new Handler("/", "", 0, new SubversionRepository());
         processStream(new ByteArrayInputStream(buffer.getBytes("UTF-8")));
-        return new History(handler.entries);
+        return new History(handler.entries, handler.getRenamedFiles());
     }
 }