Skip to content

Commit b80fbbe

Browse files
lorbanlorban
committed
nodefs conversion first steps
Signed-off-by: Ludovic Orban <[email protected]>
1 parent ff07d74 commit b80fbbe

File tree

7 files changed

+92
-36
lines changed

7 files changed

+92
-36
lines changed

‎src/main/java/org/mortbay/jetty/orchestrator/NodeArray.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ public Path rootPathOf(String id)
5959
}
6060
else
6161
{
62-
URI uri = URI.create(NodeFileSystemProvider.PREFIX + ":" + node.globalNodeId.getHostId() + "!/." + NodeFileSystemProvider.PREFIX + "/" + node.globalNodeId.getNodeId());
62+
URI uri = URI.create(NodeFileSystemProvider.SCHEME + ":" + node.globalNodeId.getHostId() + "!/." + NodeFileSystemProvider.SCHEME + "/" + node.globalNodeId.getNodeId());
6363
return Paths.get(uri);
6464
}
6565
}

‎src/main/java/org/mortbay/jetty/orchestrator/configuration/LocalHostLauncher.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ public void close() throws Exception
114114

115115
public static File rootPathOf(String hostId)
116116
{
117-
return new File(System.getProperty("user.home") + "/." + NodeFileSystemProvider.PREFIX + "/" + hostId);
117+
return new File(System.getProperty("user.home") + "/." + NodeFileSystemProvider.SCHEME + "/" + hostId);
118118
}
119119

120120
private static void copyFile(String hostId, String filename, InputStream contents) throws Exception

‎src/main/java/org/mortbay/jetty/orchestrator/configuration/SshRemoteHostLauncher.java‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -149,9 +149,13 @@ public String launch(GlobalNodeId globalNodeId, String connectString) throws Exc
149149

150150
// create remote filesystem
151151
HashMap<String, Object> env = new HashMap<>();
152-
env.put(SftpClient.class.getName(), SftpClientFactory.instance().createSftpClient(session));
153-
env.put(NodeFileSystemProvider.IS_WINDOWS_ENV_PROPERTY, windows);
154-
fileSystem = FileSystems.newFileSystem(URI.create(NodeFileSystemProvider.PREFIX + ":" + nodeId.getHostId()), env);
152+
env.put(SshClient.class.getName(), sshClient);
153+
env.put(NodeFileSystemProvider.SFTP_HOST_ENV, nodeId.getHostname());
154+
env.put(NodeFileSystemProvider.SFTP_PORT_ENV, port);
155+
env.put(NodeFileSystemProvider.SFTP_USERNAME_ENV, username);
156+
env.put(NodeFileSystemProvider.SFTP_PASSWORD_ENV, password);
157+
env.put(NodeFileSystemProvider.IS_WINDOWS_ENV, windows);
158+
fileSystem = FileSystems.newFileSystem(URI.create(NodeFileSystemProvider.SCHEME + ":" + nodeId.getHostId()), env);
155159

156160
// upload classpath
157161
List<String> remoteClasspathEntries = new ArrayList<>();
@@ -164,14 +168,14 @@ public String launch(GlobalNodeId globalNodeId, String connectString) throws Exc
164168
File cpFile = new File(classpathEntry);
165169
String cpFileName = cpFile.getName();
166170
if (!cpFileName.toLowerCase(Locale.ROOT).endsWith(".jar"))
167-
remoteClasspathEntries.add("." + NodeFileSystemProvider.PREFIX + delimiter + nodeId.getHostId() + delimiter + NodeProcess.CLASSPATH_FOLDER_NAME + delimiter + cpFileName);
171+
remoteClasspathEntries.add("." + NodeFileSystemProvider.SCHEME + delimiter + nodeId.getHostId() + delimiter + NodeProcess.CLASSPATH_FOLDER_NAME + delimiter + cpFileName);
168172
if (cpFile.isDirectory())
169173
copyDir(sftpClient, nodeId.getHostId(), cpFile, 1);
170174
else
171175
copyFile(sftpClient, nodeId.getHostId(), cpFileName, cpFile);
172176
}
173177
}
174-
remoteClasspathEntries.add("." + NodeFileSystemProvider.PREFIX + delimiter + nodeId.getHostId() + delimiter + NodeProcess.CLASSPATH_FOLDER_NAME + delimiter + "*");
178+
remoteClasspathEntries.add("." + NodeFileSystemProvider.SCHEME + delimiter + nodeId.getHostId() + delimiter + NodeProcess.CLASSPATH_FOLDER_NAME + delimiter + "*");
175179

176180
// spawn remote node jvm
177181
String cmdLine = String.join(" ", buildCommandLine(fileSystem, jvm, remoteClasspathEntries, windows ? ";" : ":", nodeId.getHostId(), nodeId.getHostname(), remoteConnectString));
@@ -249,7 +253,7 @@ private static List<String> filterOutEmptyStrings(List<String> opts)
249253

250254
private static void copyFile(SftpClient sftpClient, String hostId, String filename, File localSourceFile) throws Exception
251255
{
252-
String destFilename = "." + NodeFileSystemProvider.PREFIX + "/" + hostId + "/" + NodeProcess.CLASSPATH_FOLDER_NAME + "/" + filename;
256+
String destFilename = "." + NodeFileSystemProvider.SCHEME + "/" + hostId + "/" + NodeProcess.CLASSPATH_FOLDER_NAME + "/" + filename;
253257

254258
try (OutputStream os = sftpClient.write(destFilename);
255259
FileInputStream is = new FileInputStream(localSourceFile))
@@ -335,7 +339,7 @@ public void close() throws Exception
335339
{
336340
try (SftpClient sftpClient = SftpClientFactory.instance().createSftpClient(session))
337341
{
338-
deltree(sftpClient, "." + NodeFileSystemProvider.PREFIX + "/" + nodeId.getClusterId());
342+
deltree(sftpClient, "." + NodeFileSystemProvider.SCHEME + "/" + nodeId.getClusterId());
339343
}
340344
}
341345
IOUtil.close(forwarding);

‎src/main/java/org/mortbay/jetty/orchestrator/nodefs/NodeFileSystem.java‎

Lines changed: 36 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,14 @@
2222
import java.nio.file.WatchService;
2323
import java.nio.file.attribute.UserPrincipalLookupService;
2424
import java.nio.file.spi.FileSystemProvider;
25+
import java.util.ArrayList;
2526
import java.util.Collections;
2627
import java.util.List;
2728
import java.util.Set;
2829

2930
import org.apache.sshd.client.SshClient;
3031
import org.apache.sshd.sftp.client.fs.SftpFileSystemProvider;
32+
import org.apache.sshd.sftp.common.SftpConstants;
3133
import org.slf4j.Logger;
3234
import org.slf4j.LoggerFactory;
3335

@@ -39,27 +41,43 @@ class NodeFileSystem extends FileSystem
3941
private final NodeFileSystemProvider provider;
4042
private final String hostId;
4143
private final boolean windows;
44+
private final NodePath homePath;
45+
private final NodePath cwdPath;
4246
private final FileSystem delegate;
4347

44-
NodeFileSystem(NodeFileSystemProvider provider, SshClient sshClient, String hostId, List<String> cwd, boolean windows)
48+
NodeFileSystem(NodeFileSystemProvider provider, SshClient sshClient, String hostId, String cwd, boolean windows, String sftpHost, int sftpPort, String sftpUsername, char[] sftpPassword)
4549
{
4650
this.provider = provider;
4751
this.hostId = hostId;
4852
this.windows = windows;
4953

5054
SftpFileSystemProvider sftpFileSystemProvider = new SftpFileSystemProvider(sshClient);
51-
URI uri = SftpFileSystemProvider.createFileSystemURI("localhost", 22, "lorban", null);
55+
56+
57+
URI uri = SftpFileSystemProvider.createFileSystemURI(sftpHost, sftpPort, sftpUsername, sftpPassword == null || sftpPassword.length == 0 ? null : new String(sftpPassword));
5258

5359
try
5460
{
61+
//String userAuth = SftpFileSystemProvider.encodeCredentials(sftpUsername, sftpPassword == null || sftpPassword.length == 0 ? null : new String(sftpPassword));
62+
//URI uri = new URI(SftpConstants.SFTP_SUBSYSTEM_NAME, userAuth, sftpHost, sftpPort, cwd, null, null);
5563
delegate = sftpFileSystemProvider.newFileSystem(uri, Collections.emptyMap());
64+
this.homePath = new NodePath(this, null, NodePath.toSegments(delegate.getPath(".").toAbsolutePath().normalize().toString()));
65+
this.cwdPath = new NodePath(this, homePath, NodePath.toSegments(cwd));
5666
}
5767
catch (IOException e)
5868
{
5969
throw new RuntimeException(e);
6070
}
6171
}
6272

73+
public Path delegatePath(NodePath path)
74+
{
75+
if (!path.isAbsolute())
76+
return delegate.getPath(cwdPath.resolve(path).toAbsolutePath().toString());
77+
else
78+
return delegate.getPath(path.toAbsolutePath().toString());
79+
}
80+
6381
boolean isWindows()
6482
{
6583
return windows;
@@ -76,6 +94,11 @@ public FileSystemProvider provider()
7694
return provider;
7795
}
7896

97+
public FileSystemProvider delegateProvider()
98+
{
99+
return delegate.provider();
100+
}
101+
79102
@Override
80103
public void close() throws IOException
81104
{
@@ -103,7 +126,7 @@ public String getSeparator()
103126
@Override
104127
public Iterable<Path> getRootDirectories()
105128
{
106-
return delegate.getRootDirectories();
129+
return Collections.singleton(new NodePath(this, null, Collections.emptyList()));
107130
}
108131

109132
@Override
@@ -121,7 +144,16 @@ public Set<String> supportedFileAttributeViews()
121144
@Override
122145
public Path getPath(String first, String... more)
123146
{
124-
return delegate.getPath(first, more);
147+
boolean absolute = first.startsWith(PATH_SEPARATOR);
148+
List<String> segments = new ArrayList<>(NodePath.toSegments(first));
149+
for (String s : more)
150+
segments.addAll(NodePath.toSegments(s));
151+
return getPath(absolute, segments);
152+
}
153+
154+
Path getPath(boolean absolute, List<String> segments)
155+
{
156+
return cwdPath.resolve(absolute, segments);
125157
}
126158

127159
@Override

‎src/main/java/org/mortbay/jetty/orchestrator/nodefs/NodeFileSystemProvider.java‎

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import java.nio.file.LinkOption;
2828
import java.nio.file.OpenOption;
2929
import java.nio.file.Path;
30+
import java.nio.file.ProviderMismatchException;
3031
import java.nio.file.ReadOnlyFileSystemException;
3132
import java.nio.file.attribute.BasicFileAttributes;
3233
import java.nio.file.attribute.FileAttribute;
@@ -47,8 +48,13 @@
4748
*/
4849
public class NodeFileSystemProvider extends FileSystemProvider
4950
{
50-
public static final String PREFIX = "jco";
51-
public static final String IS_WINDOWS_ENV_PROPERTY = "windows";
51+
public static final String SCHEME = "jco";
52+
public static final String IS_WINDOWS_ENV = "windows";
53+
public static final String SFTP_HOST_ENV = "host";
54+
public static final String SFTP_PORT_ENV = "port";
55+
public static final String SFTP_USERNAME_ENV = "username";
56+
public static final String SFTP_PASSWORD_ENV = "password";
57+
5258
private static final Map<AccessMode, Integer> ACCESS_MODES_MASKS = new EnumMap<>(AccessMode.class);
5359
static
5460
{
@@ -98,13 +104,17 @@ public FileSystem newFileSystem(URI uri, Map<String, ?> env)
98104
{
99105
synchronized (fileSystems)
100106
{
101-
boolean windows = (Boolean)env.get(IS_WINDOWS_ENV_PROPERTY);
107+
boolean windows = (Boolean)env.get(IS_WINDOWS_ENV);
108+
String sftpHost = (String)env.get(SFTP_HOST_ENV);
109+
Integer sftpPort = (Integer)env.get(SFTP_PORT_ENV);
110+
String sftpUsername = (String)env.get(SFTP_USERNAME_ENV);
111+
char[] sftpPassword = (char[])env.get(SFTP_PASSWORD_ENV);
102112
SshClient sshClient = (SshClient)env.get(SshClient.class.getName());
103113
String hostId = extractHostId(uri);
104114
if (fileSystems.containsKey(hostId))
105115
throw new FileSystemAlreadyExistsException("FileSystem already exists: " + hostId);
106116

107-
NodeFileSystem fileSystem = new NodeFileSystem(this, sshClient, hostId, extractPath(uri), windows);
117+
NodeFileSystem fileSystem = new NodeFileSystem(this, sshClient, hostId, extractPath(uri), windows, sftpHost, sftpPort, sftpUsername, sftpPassword);
108118
fileSystems.put(hostId, fileSystem);
109119
return fileSystem;
110120
}
@@ -140,13 +150,13 @@ private static String extractHostId(URI uri)
140150
return nodeId;
141151
}
142152

143-
private static List<String> extractPath(URI uri)
153+
private static String extractPath(URI uri)
144154
{
145155
String nodeId = uri.getSchemeSpecificPart();
146156
int i = nodeId.indexOf("!/");
147157
if (i == -1)
148-
return Collections.emptyList();
149-
return NodePath.toSegments(nodeId.substring(i + 1));
158+
return "";
159+
return nodeId.substring(i + 1);
150160
}
151161

152162
private static String extractPathAsString(URI uri)
@@ -174,7 +184,7 @@ public Path getPath(URI uri)
174184
@Override
175185
public String getScheme()
176186
{
177-
return PREFIX;
187+
return SCHEME;
178188
}
179189

180190
@Override
@@ -210,7 +220,11 @@ public SeekableByteChannel newByteChannel(Path path, Set<? extends OpenOption> o
210220
@Override
211221
public DirectoryStream<Path> newDirectoryStream(Path dir, DirectoryStream.Filter<? super Path> filter) throws IOException
212222
{
213-
return dir.getFileSystem().provider().newDirectoryStream(dir, filter);
223+
if (!(dir instanceof NodePath))
224+
throw new ProviderMismatchException("Path is not NodePath: " + dir.getClass());
225+
NodeFileSystem delegateFileSystem = (NodeFileSystem)dir.getFileSystem();
226+
Path path = delegateFileSystem.delegatePath((NodePath) dir);
227+
return delegateFileSystem.delegateProvider().newDirectoryStream(path, filter);
214228
}
215229

216230
@Override

‎src/main/java/org/mortbay/jetty/orchestrator/rpc/NodeProcess.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -192,7 +192,7 @@ public static Thread spawnThread(String nodeId, String connectString)
192192

193193
private static File defaultRootPath(String hostId)
194194
{
195-
return new File(System.getProperty("user.home") + "/." + NodeFileSystemProvider.PREFIX + "/" + hostId);
195+
return new File(System.getProperty("user.home") + "/." + NodeFileSystemProvider.SCHEME + "/" + hostId);
196196
}
197197

198198
private static File defaultLibPath(String hostId)

‎src/test/java/org/mortbay/jetty/orchestrator/nodefs/NodeFileSystemTest.java‎

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -64,19 +64,25 @@ public void tearDown()
6464
@Test
6565
public void testNodeIdFolder() throws Exception
6666
{
67-
new File("target/testNodeIdFolder/." + NodeFileSystemProvider.PREFIX + "/the-test/myhost/a").mkdirs();
67+
new File("target/testNodeIdFolder/." + NodeFileSystemProvider.SCHEME + "/the-test/myhost/a").mkdirs();
6868

6969
TestSshServer testSshServer = closer.register(new TestSshServer("target/testNodeIdFolder"));
70-
SshClient sshClient = closer.register(new SshClient());
70+
SshClient sshClient = closer.register(SshClient.setUpDefaultClient());
7171
sshClient.start();
7272
closer.register(sshClient.connect(null, "localhost", testSshServer.getPort())
7373
.verify(30, TimeUnit.SECONDS)
7474
.getSession());
7575

7676
HashMap<String, Object> env = new HashMap<>();
77-
env.put(NodeFileSystemProvider.IS_WINDOWS_ENV_PROPERTY, IS_WINDOWS);
77+
env.put(NodeFileSystemProvider.IS_WINDOWS_ENV, IS_WINDOWS);
78+
env.put(NodeFileSystemProvider.SFTP_HOST_ENV, "localhost");
79+
env.put(NodeFileSystemProvider.SFTP_PORT_ENV, testSshServer.getPort());
80+
env.put(NodeFileSystemProvider.SFTP_USERNAME_ENV, System.getProperty("user.name"));
7881
env.put(SshClient.class.getName(), sshClient);
79-
NodeFileSystem fileSystem = closer.register((NodeFileSystem)FileSystems.newFileSystem(URI.create(NodeFileSystemProvider.PREFIX + ":the-test/myhost!/." + NodeFileSystemProvider.PREFIX + "/the-test/myhost"), env));
82+
NodeFileSystem fileSystem = closer.register((NodeFileSystem)FileSystems.newFileSystem(URI.create(NodeFileSystemProvider.SCHEME + ":the-test/myhost!/." + NodeFileSystemProvider.SCHEME + "/the-test/myhost"), env));
83+
84+
Files.newDirectoryStream(fileSystem.getPath(".")).forEach((s) -> System.out.println(s));
85+
Files.newDirectoryStream(fileSystem.getPath(".jco")).forEach((s) -> System.out.println(s));
8086

8187
DirectoryStream<Path> paths = Files.newDirectoryStream(fileSystem.getPath("."));
8288
Iterator<Path> iterator = paths.iterator();
@@ -88,7 +94,7 @@ public void testNodeIdFolder() throws Exception
8894
@Test
8995
public void testHomeFolderIsDefault() throws Exception
9096
{
91-
new File("target/testHomeFolderIsDefault/." + NodeFileSystemProvider.PREFIX + "/the-test/myhost").mkdirs();
97+
new File("target/testHomeFolderIsDefault/." + NodeFileSystemProvider.SCHEME + "/the-test/myhost").mkdirs();
9298

9399
TestSshServer testSshServer = closer.register(new TestSshServer("target/testHomeFolderIsDefault"));
94100
SshClient sshClient = closer.register(new SshClient());
@@ -98,9 +104,9 @@ public void testHomeFolderIsDefault() throws Exception
98104
.getSession());
99105

100106
HashMap<String, Object> env = new HashMap<>();
101-
env.put(NodeFileSystemProvider.IS_WINDOWS_ENV_PROPERTY, IS_WINDOWS);
107+
env.put(NodeFileSystemProvider.IS_WINDOWS_ENV, IS_WINDOWS);
102108
env.put(SshClient.class.getName(), sshClient);
103-
FileSystem fileSystem = closer.register(FileSystems.newFileSystem(URI.create(NodeFileSystemProvider.PREFIX + ":the-test/myhost"), env));
109+
FileSystem fileSystem = closer.register(FileSystems.newFileSystem(URI.create(NodeFileSystemProvider.SCHEME + ":the-test/myhost"), env));
104110

105111
DirectoryStream<Path> paths = Files.newDirectoryStream(fileSystem.getPath("."));
106112
Iterator<Path> iterator = paths.iterator();
@@ -122,9 +128,9 @@ public void testAbsolutePath() throws Exception
122128
.getSession());
123129

124130
HashMap<String, Object> env = new HashMap<>();
125-
env.put(NodeFileSystemProvider.IS_WINDOWS_ENV_PROPERTY, IS_WINDOWS);
131+
env.put(NodeFileSystemProvider.IS_WINDOWS_ENV, IS_WINDOWS);
126132
env.put(SshClient.class.getName(), sshClient);
127-
FileSystem fileSystem = closer.register(FileSystems.newFileSystem(URI.create(NodeFileSystemProvider.PREFIX + ":the-test/myhost"), env));
133+
FileSystem fileSystem = closer.register(FileSystems.newFileSystem(URI.create(NodeFileSystemProvider.SCHEME + ":the-test/myhost"), env));
128134

129135
DirectoryStream<Path> directoryStream = Files.newDirectoryStream(fileSystem.getPath("/"));
130136
long pathCount = StreamSupport.stream(Spliterators.spliteratorUnknownSize(directoryStream.iterator(), Spliterator.ORDERED), false).count();
@@ -150,9 +156,9 @@ public void testJvmFilenameSupplierFound() throws Exception
150156
.getSession());
151157

152158
HashMap<String, Object> env = new HashMap<>();
153-
env.put(NodeFileSystemProvider.IS_WINDOWS_ENV_PROPERTY, IS_WINDOWS);
159+
env.put(NodeFileSystemProvider.IS_WINDOWS_ENV, IS_WINDOWS);
154160
env.put(SshClient.class.getName(), sshClient);
155-
FileSystem fileSystem = closer.register(FileSystems.newFileSystem(URI.create(NodeFileSystemProvider.PREFIX + ":the-test/myhost"), env));
161+
FileSystem fileSystem = closer.register(FileSystems.newFileSystem(URI.create(NodeFileSystemProvider.SCHEME + ":the-test/myhost"), env));
156162

157163
Jvm jvm = new Jvm((fs, h) ->
158164
{
@@ -188,9 +194,9 @@ public void testJvmFilenameSupplierNotFound() throws Exception
188194
.getSession());
189195

190196
HashMap<String, Object> env = new HashMap<>();
191-
env.put(NodeFileSystemProvider.IS_WINDOWS_ENV_PROPERTY, IS_WINDOWS);
197+
env.put(NodeFileSystemProvider.IS_WINDOWS_ENV, IS_WINDOWS);
192198
env.put(SshClient.class.getName(), sshClient);
193-
FileSystem fileSystem = closer.register(FileSystems.newFileSystem(URI.create(NodeFileSystemProvider.PREFIX + ":the-test/myhost"), env));
199+
FileSystem fileSystem = closer.register(FileSystems.newFileSystem(URI.create(NodeFileSystemProvider.SCHEME + ":the-test/myhost"), env));
194200

195201
assertThrows(NoFileException.class, () -> new Jvm((fs, h) ->
196202
{

0 commit comments

Comments
 (0)