#83 Add property s3fs_path_style_access to allow to customize the url

jarnaiz · jarnaiz · commit 1e79b151fda7 · 2017-12-11T17:16:38.000+01:00
diff --git a/README.md b/README.md
@@ -14,7 +14,7 @@ This project provides a first API implementation, little optimized, but "complet
 <dependency>
 	<groupId>com.upplication</groupId>
 	<artifactId>s3fs</artifactId>
-	<version>2.0.0</version>
+	<version>2.2.0</version>
 </dependency>
 ```
 
@@ -79,6 +79,8 @@ Complete settings lists:
 * s3fs_socket_timeout
 * s3fs_user_agent
 * s3fs_amazon_s3_factory
+* s3fs_signer_override
+* s3fs_path_style_access
 
 ##### Set endpoint to reduce data latency in your applications
 
@@ -138,7 +140,6 @@ Now you can inject in any spring component:
 ```java
 @Autowired
 private FileSystem s3FileSystem;
-
 ```
 
 ##### What is new 2.0.0
@@ -147,6 +148,7 @@ private FileSystem s3FileSystem;
 * Removed META-INF/services/java.nio.file.spi.FileTypeDetector #78
 * Bucket are filestores and root directories for a bucket is the bucket itself.
 * getFileName for a root Path is ```null```
+* Added new method toURL to S3Path and can be customized with the property s3fs_path_style_access #83
 * Improved S3Path Tests
 
 #### Features:
diff --git a/pom.xml b/pom.xml
@@ -4,7 +4,7 @@
 	<groupId>com.upplication</groupId>
 	<artifactId>s3fs</artifactId>
 	<packaging>jar</packaging>
-	<version>2.1.0</version>
+	<version>2.2.0</version>
 	<name>s3fs</name>
 	<description>S3 filesystem provider for Java 7</description>
 	<url>https://github.com/Upplication/Amazon-S3-FileSystem-NIO2</url>
diff --git a/src/main/java/com/upplication/s3fs/AmazonS3Factory.java b/src/main/java/com/upplication/s3fs/AmazonS3Factory.java
@@ -8,10 +8,12 @@
 import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
 import com.amazonaws.metrics.RequestMetricCollector;
 import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.S3ClientOptions;
 
 import java.net.URI;
 import java.util.Properties;
 
+
 /**
  * Factory base class to create a new AmazonS3 instance.
  */
@@ -35,6 +37,7 @@ public abstract class AmazonS3Factory {
     public static final String SOCKET_TIMEOUT = "s3fs_socket_timeout";
     public static final String USER_AGENT = "s3fs_user_agent";
     public static final String SIGNER_OVERRIDE = "s3fs_signer_override";
+    public static final String PATH_STYLE_ACCESS = "s3fs_path_style_access";
 
     /**
      * Build a new Amazon S3 instance with the URI and the properties provided
@@ -50,6 +53,9 @@ public AmazonS3 getAmazonS3(URI uri, Properties props) {
             else
                 client.setEndpoint(uri.getHost());
         }
+
+        client.setS3ClientOptions(getClientOptions(props));
+
         return client;
     }
 
@@ -84,6 +90,15 @@ protected RequestMetricCollector getRequestMetricsCollector(Properties props) {
         return requestMetricCollector;
     }
 
+    protected S3ClientOptions getClientOptions(Properties props) {
+        S3ClientOptions.Builder builder = S3ClientOptions.builder();
+        if (props.getProperty(PATH_STYLE_ACCESS) != null &&
+                Boolean.parseBoolean(props.getProperty(PATH_STYLE_ACCESS)))
+            builder.setPathStyleAccess(true);
+
+        return builder.build();
+    }
+
     protected ClientConfiguration getClientConfiguration(Properties props) {
         ClientConfiguration clientConfiguration = new ClientConfiguration();
         if (props.getProperty(CONNECTION_TIMEOUT) != null)
diff --git a/src/main/java/com/upplication/s3fs/S3FileSystemProvider.java b/src/main/java/com/upplication/s3fs/S3FileSystemProvider.java
@@ -71,7 +71,7 @@ public class S3FileSystemProvider extends FileSystemProvider {
     private static final ConcurrentMap<String, S3FileSystem> fileSystems = new ConcurrentHashMap<>();
     private static final List<String> PROPS_TO_OVERLOAD = Arrays.asList(ACCESS_KEY, SECRET_KEY, REQUEST_METRIC_COLLECTOR_CLASS, CONNECTION_TIMEOUT, MAX_CONNECTIONS, MAX_ERROR_RETRY, PROTOCOL, PROXY_DOMAIN,
             PROXY_HOST, PROXY_PASSWORD, PROXY_PORT, PROXY_USERNAME, PROXY_WORKSTATION, SOCKET_SEND_BUFFER_SIZE_HINT, SOCKET_RECEIVE_BUFFER_SIZE_HINT, SOCKET_TIMEOUT,
-            USER_AGENT, AMAZON_S3_FACTORY_CLASS);
+            USER_AGENT, AMAZON_S3_FACTORY_CLASS, SIGNER_OVERRIDE, PATH_STYLE_ACCESS);
 
     private S3Utils s3Utils = new S3Utils();
     private Cache cache = new Cache();
diff --git a/src/main/java/com/upplication/s3fs/S3Path.java b/src/main/java/com/upplication/s3fs/S3Path.java
@@ -9,6 +9,7 @@
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.net.URI;
+import java.net.URL;
 import java.net.URLDecoder;
 import java.nio.file.*;
 import java.util.Iterator;
@@ -60,8 +61,8 @@ public S3Path(S3FileSystem fileSystem, String first, String... more) {
 
         List<String> pathsURI = Lists
                 .newArrayList(Splitter.on(PATH_SEPARATOR)
-                .omitEmptyStrings()
-                .split(first));
+                        .omitEmptyStrings()
+                        .split(first));
 
         if (hasBucket) { // absolute path
 
@@ -468,6 +469,26 @@ public URI toUri() {
         }
     }
 
+    /**
+     * Get the url for the s3Path.
+     *
+     * The url represents a Uniform Resource
+     * Locator, a pointer to a "resource" on the World
+     * Wide Web.
+     *
+     * All S3Path has a URL if is absolute
+     *
+     * @see com.amazonaws.services.s3.AmazonS3#getUrl(String, String)
+     * @see S3Path#toUri() for unique resource identifier
+     * @return URL or null if is not absoulte
+     */
+    public URL toURL() {
+        if (!this.isAbsolute())
+            return null;
+
+        return this.getFileSystem().getClient().getUrl(this.fileStore.name(), this.getKey());
+    }
+
     @Override
     public Path toAbsolutePath() {
         if (isAbsolute()) {
@@ -507,7 +528,7 @@ public Iterator<Path> iterator() {
         }
 
         List<String> paths = uriToList();
-        String lastPath = paths.get(paths.size()-1);
+        String lastPath = paths.get(paths.size() - 1);
 
         for (String path : uriToList()) {
             String pathFinal = path + PATH_SEPARATOR;
diff --git a/src/test/java/com/upplication/s3fs/AmazonS3ClientFactoryTest.java b/src/test/java/com/upplication/s3fs/AmazonS3ClientFactoryTest.java
@@ -3,9 +3,7 @@
 import static com.amazonaws.SDKGlobalConfiguration.ACCESS_KEY_SYSTEM_PROPERTY;
 import static com.amazonaws.SDKGlobalConfiguration.SECRET_KEY_SYSTEM_PROPERTY;
 import static com.upplication.s3fs.AmazonS3Factory.*;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
 
 import java.net.URI;
 import java.util.Properties;
@@ -43,6 +41,7 @@ public void neverTrustTheDefaults() {
         props.setProperty(SOCKET_TIMEOUT, "30");
         props.setProperty(USER_AGENT, "I-am-Groot");
         props.setProperty(SIGNER_OVERRIDE, "S3SignerType");
+        props.setProperty(PATH_STYLE_ACCESS, "true");
         ExposingAmazonS3Client client = (ExposingAmazonS3Client) clientFactory.getAmazonS3(S3EndpointConstant.S3_GLOBAL_URI_TEST, props);
         AWSCredentialsProvider credentialsProvider = client.getAWSCredentialsProvider();
         AWSCredentials credentials = credentialsProvider.getCredentials();
@@ -65,6 +64,7 @@ public void neverTrustTheDefaults() {
         assertEquals(30, clientConfiguration.getSocketTimeout());
         assertEquals("I-am-Groot", clientConfiguration.getUserAgent());
         assertEquals("S3SignerType", clientConfiguration.getSignerOverride());
+        assertTrue(client.getClientOptions().isPathStyleAccess());
     }
 
     @Test
@@ -95,6 +95,7 @@ public void theDefaults() {
         assertEquals(50000, clientConfiguration.getSocketTimeout());
         assertTrue(clientConfiguration.getUserAgent().startsWith("aws-sdk-java"));
         assertNull(clientConfiguration.getSignerOverride());
+        assertFalse(client.getClientOptions().isPathStyleAccess());
     }
 
     @Test(expected = IllegalArgumentException.class)
diff --git a/src/test/java/com/upplication/s3fs/AmazonS3ClientIT.java b/src/test/java/com/upplication/s3fs/AmazonS3ClientIT.java
@@ -5,7 +5,6 @@
 import static com.upplication.s3fs.util.EnvironmentBuilder.getRealEnv;
 import static java.util.UUID.randomUUID;
 import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.fail;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
@@ -15,11 +14,9 @@
 import java.util.Map;
 
 import org.junit.Before;
-import org.junit.Ignore;
 
 import com.amazonaws.auth.BasicAWSCredentials;
 import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.model.AmazonS3Exception;
 import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.amazonaws.services.s3.model.PutObjectResult;
 import com.upplication.s3fs.util.EnvironmentBuilder;
diff --git a/src/test/java/com/upplication/s3fs/FileSystemProvider/DeleteTest.java b/src/test/java/com/upplication/s3fs/FileSystemProvider/DeleteTest.java
@@ -13,13 +13,10 @@
 import java.io.IOException;
 import java.net.URI;
 import java.nio.file.*;
-import java.util.Properties;
 
 import static com.upplication.s3fs.AmazonS3Factory.ACCESS_KEY;
 import static com.upplication.s3fs.AmazonS3Factory.SECRET_KEY;
-import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.*;
 
 public class DeleteTest extends S3UnitTestBase {
 
diff --git a/src/test/java/com/upplication/s3fs/FileSystemProvider/GetFileSystemIT.java b/src/test/java/com/upplication/s3fs/FileSystemProvider/GetFileSystemIT.java
@@ -4,10 +4,8 @@
 import com.upplication.s3fs.S3FileSystemProvider;
 import org.junit.Before;
 import org.junit.Test;
-import org.mockito.ArgumentMatcher;
 
 import java.io.IOException;
-import java.net.URI;
 import java.nio.file.FileSystem;
 import java.nio.file.FileSystemNotFoundException;
 import java.nio.file.FileSystems;
@@ -18,8 +16,6 @@
 import static com.upplication.s3fs.AmazonS3Factory.SECRET_KEY;
 import static com.upplication.s3fs.util.S3EndpointConstant.S3_GLOBAL_URI_IT;
 import static org.junit.Assert.*;
-import static org.mockito.Matchers.argThat;
-import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.*;
 
 public class GetFileSystemIT {
diff --git a/src/test/java/com/upplication/s3fs/FileSystemProvider/NewFileSystemTest.java b/src/test/java/com/upplication/s3fs/FileSystemProvider/NewFileSystemTest.java
@@ -14,7 +14,6 @@
 import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.doCallRealMethod;
 import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.verify;
 
 import java.io.IOException;
diff --git a/src/test/java/com/upplication/s3fs/FileSystemProvider/NewInputStreamTest.java b/src/test/java/com/upplication/s3fs/FileSystemProvider/NewInputStreamTest.java
@@ -16,14 +16,11 @@
 import java.net.URI;
 import java.nio.file.*;
 import java.util.Map;
-import java.util.Properties;
 
 import static com.upplication.s3fs.AmazonS3Factory.ACCESS_KEY;
 import static com.upplication.s3fs.AmazonS3Factory.SECRET_KEY;
 import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
-import static org.mockito.Mockito.*;
 
 public class NewInputStreamTest extends S3UnitTestBase {
 
diff --git a/src/test/java/com/upplication/s3fs/FileSystemProvider/NewOutputStreamTest.java b/src/test/java/com/upplication/s3fs/FileSystemProvider/NewOutputStreamTest.java
@@ -14,13 +14,10 @@
 import java.io.OutputStream;
 import java.net.URI;
 import java.nio.file.*;
-import java.util.Properties;
 
 import static com.upplication.s3fs.AmazonS3Factory.ACCESS_KEY;
 import static com.upplication.s3fs.AmazonS3Factory.SECRET_KEY;
 import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.*;
 
 public class NewOutputStreamTest extends S3UnitTestBase {
 
diff --git a/src/test/java/com/upplication/s3fs/Path/ToURLIT.java b/src/test/java/com/upplication/s3fs/Path/ToURLIT.java
@@ -0,0 +1,97 @@
+package com.upplication.s3fs.Path;
+
+import com.upplication.s3fs.*;
+import com.upplication.s3fs.util.EnvironmentBuilder;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URL;
+import java.nio.file.*;
+import java.util.HashMap;
+import java.util.Map;
+
+import static com.upplication.s3fs.util.S3EndpointConstant.S3_GLOBAL_URI_IT;
+import static com.upplication.s3fs.AmazonS3ClientFactory.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+public class ToURLIT {
+
+    private static final URI uriGlobal = EnvironmentBuilder.getS3URI(S3_GLOBAL_URI_IT);
+
+    public FileSystem getS3FileSystem(Entry... props) throws IOException {
+
+        System.clearProperty(S3FileSystemProvider.AMAZON_S3_FACTORY_CLASS);
+
+        Map<String, Object> env = new HashMap<>(EnvironmentBuilder.getRealEnv());
+        if (props != null) {
+            for (Entry entry : props) {
+                env.put(entry.getKey(), entry.getValue());
+            }
+        }
+
+        try {
+            return FileSystems.newFileSystem(uriGlobal, env);
+        } catch (FileSystemAlreadyExistsException e) {
+            FileSystems.getFileSystem(uriGlobal).close();
+            return FileSystems.newFileSystem(uriGlobal, env);
+        }
+    }
+
+    @Test
+    public void toURLDefault() throws IOException {
+
+        FileSystem fs = getS3FileSystem();
+
+        S3Path s3Path = (S3Path) fs.getPath("/bucket.with.dots").resolve("index.html");
+
+        assertEquals(new URL("https://bucket.with.dots." + S3_GLOBAL_URI_IT.toString().replace("s3://", "") + "index.html"), s3Path.toURL());
+    }
+
+    @Test
+    public void toURLWithPathStyle() throws IOException {
+
+        FileSystem fs = getS3FileSystem(new Entry(PATH_STYLE_ACCESS, "true"));
+
+        S3Path s3Path = (S3Path) fs.getPath("/bucket.with.dots").resolve("index.html");
+
+        assertEquals(new URL("https://" + S3_GLOBAL_URI_IT.toString().replace("s3://", "") + "bucket.with.dots/index.html"), s3Path.toURL());
+    }
+
+    @Test
+    public void toURLNull() throws IOException {
+
+        FileSystem fs = getS3FileSystem(new Entry(PATH_STYLE_ACCESS, "true"));
+
+        S3Path s3Path = (S3Path) fs.getPath("directory").resolve("index.html");
+
+        assertNull(s3Path.toURL());
+    }
+
+    public static class Entry {
+        private String key;
+        private String value;
+
+        public Entry(String key, String value) {
+            this.key = key;
+            this.value = value;
+        }
+
+        public String getKey() {
+            return key;
+        }
+
+        public void setKey(String key) {
+            this.key = key;
+        }
+
+        public String getValue() {
+            return value;
+        }
+
+        public void setValue(String value) {
+            this.value = value;
+        }
+    }
+}
diff --git a/src/test/java/com/upplication/s3fs/Path/ToUriTest.java b/src/test/java/com/upplication/s3fs/Path/ToUriTest.java
diff --git a/src/test/java/com/upplication/s3fs/S3FileStoreTest.java b/src/test/java/com/upplication/s3fs/S3FileStoreTest.java
diff --git a/src/test/java/com/upplication/s3fs/S3UnitTestBase.java b/src/test/java/com/upplication/s3fs/S3UnitTestBase.java
diff --git a/src/test/java/com/upplication/s3fs/S3UtilsIT.java b/src/test/java/com/upplication/s3fs/S3UtilsIT.java
diff --git a/src/test/java/com/upplication/s3fs/spike/EnvironmentIT.java b/src/test/java/com/upplication/s3fs/spike/EnvironmentIT.java
diff --git a/src/test/java/com/upplication/s3fs/util/ExposingAmazonS3Client.java b/src/test/java/com/upplication/s3fs/util/ExposingAmazonS3Client.java
