Nekojimi/FriendCloudProto
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Initial commit (demo as distributed to Cloudy)

Browse source
This commit is contained in:
Nekojimi 2025-09-02 12:22:20 +01:00
commit fa687c2968
31 changed files with 2544 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
.gitignore vendored Normal file
View file

@ -0,0 +1,38 @@
target/
!.mvn/wrapper/maven-wrapper.jar
!**/src/main/**/target/
!**/src/test/**/target/
### IntelliJ IDEA ###
.idea/modules.xml
.idea/jarRepositories.xml
.idea/compiler.xml
.idea/libraries/
*.iws
*.iml
*.ipr
### Eclipse ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
### NetBeans ###
/nbproject/private/
/nbbuild/
/dist/
/nbdist/
/.nb-gradle/
build/
!**/src/main/**/build/
!**/src/test/**/build/
### VS Code ###
.vscode/
### Mac OS ###
.DS_Store

5
.idea/.gitignore vendored Normal file
View file

@ -0,0 +1,5 @@
# Default ignored files
/shelf/
/workspace.xml
# Environment-dependent path to Maven home directory
/mavenHomeManager.xml

5
.idea/codeStyles/codeStyleConfig.xml Normal file
View file

@ -0,0 +1,5 @@
<component name="ProjectCodeStyleConfiguration">
<state>
<option name="PREFERRED_PROJECT_CODE_STYLE" value="Default" />
</state>
</component>

7
.idea/encodings.xml Normal file
View file

@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Encoding">
<file url="file://$PROJECT_DIR$/src/main/java" charset="UTF-8" />
<file url="file://$PROJECT_DIR$/src/main/resources" charset="UTF-8" />
</component>
</project>

20
.idea/misc.xml Normal file
View file

@ -0,0 +1,20 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="EntryPointsManager">
<writeAnnotations>
<writeAnnotation name="Parameter" />
<writeAnnotation name="com.beust.jcommander.Parameter" />
</writeAnnotations>
</component>
<component name="ExternalStorageConfigurationManager" enabled="true" />
<component name="MavenProjectsManager">
<option name="originalFiles">
<list>
<option value="$PROJECT_DIR$/pom.xml" />
</list>
</option>
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_21" default="true" project-jdk-name="21" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/out" />
</component>
</project>

6
.idea/vcs.xml Normal file
View file

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>

142
pom.xml Normal file
View file

@ -0,0 +1,142 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>moe.nekojimi.friendcloud</groupId>
<artifactId>FriendCloudProto</artifactId>
<version>0.0.69</version>
<properties>
<maven.compiler.source>21</maven.compiler.source>
<maven.compiler.target>21</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<protobuf.version>3.9.2</protobuf.version>
<generated.sourceDirectory>target/generated-sources</generated.sourceDirectory>
</properties>
<dependencies>
<dependency>
<groupId>com.github.serceman</groupId>
<artifactId>jnr-fuse</artifactId>
<version>0.5.8</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>${protobuf.version}</version>
</dependency>
<dependency>
<groupId>com.offbynull.portmapper</groupId>
<artifactId>portmapper</artifactId>
<version>2.0.7-NEKOJIMI-PATCH</version>
</dependency>
<!-- <dependency>-->
<!-- <groupId>org.bitlet</groupId>-->
<!-- <artifactId>weupnp</artifactId>-->
<!-- <version>RELEASE</version>-->
<!-- </dependency>-->
<dependency>
<groupId>org.jcommander</groupId>
<artifactId>jcommander</artifactId>
<version>2.0</version>
</dependency>
<dependency>
<groupId>io.github.ascopes</groupId>
<artifactId>protobuf-maven-plugin</artifactId>
<version>3.7.0</version>
</dependency>
<dependency>
<groupId>org.jetbrains</groupId>
<artifactId>annotations</artifactId>
<version>16.0.1</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<version>2.0.17</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>2.0.17</version>
</dependency>
<dependency>
<groupId>com.kstruct</groupId>
<artifactId>gethostname4j</artifactId>
<version>1.0.0</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>io.github.ascopes</groupId>
<artifactId>protobuf-maven-plugin</artifactId>
<configuration>
<protocVersion>${protobuf.version}</protocVersion>
</configuration>
<executions>
<execution>
<goals>
<goal>generate</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- <plugin>-->
<!-- <groupId>com.github.os72</groupId>-->
<!-- <artifactId>protoc-jar-maven-plugin</artifactId>-->
<!-- <version>3.9.2</version>-->
<!-- <executions>-->
<!-- <execution>-->
<!-- <phase>generate-sources</phase>-->
<!-- <goals>-->
<!-- <goal>run</goal>-->
<!-- </goals>-->
<!-- <configuration>-->
<!-- <protocVersion>${protobuf.version}</protocVersion>-->
<!-- <inputDirectories>-->
<!-- <include>src/main/protobuf</include>-->
<!-- </inputDirectories>-->
<!-- </configuration>-->
<!-- </execution>-->
<!-- </executions>-->
<!-- </plugin>-->
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>3.3.0</version>
<configuration>
<mainClass>moe.nekojimi.friendcloud.Main</mainClass>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.6.0</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>moe.nekojimi.friendcloud.Main</mainClass>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

139
src/main/java/moe/nekojimi/friendcloud/ConnectionManager.java Normal file
View file

@ -0,0 +1,139 @@
package moe.nekojimi.friendcloud;
import moe.nekojimi.friendcloud.network.PeerConnection;
import moe.nekojimi.friendcloud.network.PeerTCPConnection;
import moe.nekojimi.friendcloud.objects.Peer;
import java.io.IOException;
import java.net.ServerSocket;
import java.net.Socket;
import java.net.URI;
import java.util.*;
import java.util.function.Consumer;
public class ConnectionManager extends Thread
{
// private final Executor executor = new ThreadPoolExecutor()
//TODO: move the TCP stuff to it's own thread, which sends NodeTCPConnections to this thread
private final ServerSocket serverSocket;
private final Set<PeerConnection> activeConnections = new HashSet<>();
private final Set<Consumer<PeerConnection>> newConnectionConsumers = new HashSet<>();
public ConnectionManager(int portNumber) throws IOException
{
serverSocket = new ServerSocket(portNumber);
// serverSocket.bind(new InetSocketAddress());
}
@Override
public void run()
{
super.run();
while (!serverSocket.isClosed())
{
try
{
Socket socket = serverSocket.accept();
System.out.println("TCP Connection Manager: accepted connection from " + socket.getRemoteSocketAddress());
PeerTCPConnection nodeTCPConnection = new PeerTCPConnection(socket);
activeConnections.add(nodeTCPConnection);
nodeTCPConnection.start();
for (Consumer<PeerConnection> consumer: newConnectionConsumers)
{
consumer.accept(nodeTCPConnection);
}
} catch (IOException e)
{
System.err.println("ConnectionManager TCP experienced exception:" + e.getMessage());
e.printStackTrace(System.err);
}
}
}
public PeerConnection getNodeConnection(URI uri) throws IOException
{
return getNodeConnection(uri, null);
}
public PeerConnection getNodeConnection(URI uri, Peer peer) throws IOException
{
purgeDeadConnections();
for (PeerConnection peerConnection: activeConnections)
{
if (peerConnection.getUri() == uri)
return peerConnection;
}
PeerConnection nodeConnection = null;
if (Objects.equals(uri.getScheme(), "tcp"))
{
nodeConnection = new PeerTCPConnection(uri, peer);
nodeConnection.start();
}
if (nodeConnection != null)
activeConnections.add(nodeConnection);
return nodeConnection;
}
public PeerConnection getNodeConnection(Peer peer) throws IOException
{
// try to find if we already have an active connection to this peer
purgeDeadConnections();
for (PeerConnection peerConnection: activeConnections)
{
if (peerConnection.getNode() == peer)
return peerConnection;
}
for (URI address : peer.getAddresses())
{
try
{
return getNodeConnection(address);
}
catch (IOException ex)
{
System.err.println("Couldn't create PeerConnection to " + address + " : " + ex.getMessage());
}
}
System.err.println("Failed to create PeerConnection to " + peer + "!");
return null;
}
public void shutdown() throws IOException
{
serverSocket.close();
for (PeerConnection nc: activeConnections)
{
nc.shutdown();
}
}
private void purgeDeadConnections()
{
Set<PeerConnection> deadConnections = new HashSet<>();
for (PeerConnection peerConnection: activeConnections)
{
if (!peerConnection.isAlive())
deadConnections.add(peerConnection);
}
activeConnections.removeAll(deadConnections);
}
public void addNewConnectionConsumer(Consumer<PeerConnection> consumer)
{
newConnectionConsumers.add(consumer);
}
public void removeNewConnectionConsumer(Consumer<PeerConnection> consumer)
{
newConnectionConsumers.remove(consumer);
}
}

138
src/main/java/moe/nekojimi/friendcloud/FileDownloadTask.java Normal file
View file

@ -0,0 +1,138 @@
package moe.nekojimi.friendcloud;
import moe.nekojimi.friendcloud.network.PeerConnection;
import moe.nekojimi.friendcloud.network.requests.FilePiecesRequest;
import moe.nekojimi.friendcloud.objects.NetworkFile;
import moe.nekojimi.friendcloud.objects.Peer;
import moe.nekojimi.friendcloud.objects.PeerFileState;
import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.*;
public class FileDownloadTask implements Callable<File>
{
private final NetworkFile file;
private final ConnectionManager manager;
private final long timeoutPerPieceMs = 10_000;
private static final int MAX_DOWNLOAD_PIECES_PER_ROUND = 128;
private final SortedSet<Integer> missingPieceIndices = new TreeSet<>();
public FileDownloadTask(NetworkFile file, ConnectionManager manager)
{
this.file = file;
this.manager = manager;
for (int i = 0; i < file.getPieceCount(); i++)
{
missingPieceIndices.add(i);
}
}
public FileDownloadTask(NetworkFile file, ConnectionManager manager, List<Integer> missingPieces)
{
this.file = file;
this.manager = manager;
missingPieceIndices.addAll(missingPieces);
}
public NetworkFile getFile()
{
return file;
}
@Override
public File call() throws Exception
{
System.out.println("Starting download of file " + file.getName());
while (!missingPieceIndices.isEmpty())
{
System.out.println("Need to get " + missingPieceIndices.size() + " missing pieces.");
Map<Peer, PeerFileState> fileStates = file.getFileStates();
// determine what nodes we can connect to
List<PeerConnection> connections = new ArrayList<>();
for (PeerFileState peerFileState : fileStates.values())
{
if (peerFileState.getProgress() >= 100.0)
{
try
{
PeerConnection connection = manager.getNodeConnection(peerFileState.getNode());
System.out.println("FileDownloadTask: Will download from " + peerFileState.getNode().getNodeName());
connections.add(connection);
} catch (IOException ex)
{
System.err.println("Failed to connect to peer " + peerFileState.getNode().getNodeName() + ": " + ex.getMessage());
}
}
}
if (connections.isEmpty())
{
System.err.println("FileDownloadTask: No peers have the file, download failed!");
return null;
}
// find a continuous run of pieces to download
// TODO: allow for runs with regular gaps (e.g. every 2) to account for previous failed download attempts
int runStart = -1;
int runEnd = -1;
for (int pieceIdx: missingPieceIndices)
{
int runLength = runEnd - runStart;
if (runLength >= MAX_DOWNLOAD_PIECES_PER_ROUND)
break;
else if (runStart == -1)
{
runStart = pieceIdx;
runEnd = pieceIdx;
}
else if (pieceIdx == runEnd + 1)
runEnd = pieceIdx;
else
break;
}
System.out.println("FileDownloadTask: Will download pieces from " + runStart + " to " + runEnd);
// make one request per connectable peer, striping the needed pieces among them
List<CompletableFuture<List<Integer>>> fileFutures = new ArrayList<>();
int offset = 0;
for (PeerConnection connection : connections)
{
CompletableFuture<List<Integer>> future = connection.makeRequest(new FilePiecesRequest(file, runStart+offset, (runEnd-runStart)+1, connections.size()));
fileFutures.add(future);
offset++;
}
long timeout = timeoutPerPieceMs * (missingPieceIndices.size() / connections.size());
// wait for all the requests to complete
for (CompletableFuture<List<Integer>> future : fileFutures)
{
try
{
List<Integer> receivedPieces = future.get(timeout, TimeUnit.MILLISECONDS);
receivedPieces.forEach(missingPieceIndices::remove);
} catch (InterruptedException e)
{
future.cancel(true);
timeout = 1_000;
System.err.println("FileDownloadTask: Request timed out.");
} catch (ExecutionException | TimeoutException e)
{
e.printStackTrace(System.err);
}
}
}
System.out.println("FileDownloadTask: finished downloading " + file.getName() + "!");
return file.getLocalFile();
}
}

75
src/main/java/moe/nekojimi/friendcloud/FilePieceAccess.java Normal file
View file

@ -0,0 +1,75 @@
package moe.nekojimi.friendcloud;
import moe.nekojimi.friendcloud.objects.NetworkFile;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
public class FilePieceAccess implements Closeable
{
private final NetworkFile networkFile;
private final File file;
private final RandomAccessFile randomAccessFile;
public FilePieceAccess(NetworkFile networkFile) throws IOException
{
this.networkFile = networkFile;
this.file = networkFile.getOrCreateLocalFile();
this.randomAccessFile = new RandomAccessFile(file,"rw");
randomAccessFile.setLength(file.length());
}
public int getPieceOffset(int index)
{
return Math.toIntExact(index * networkFile.getPieceSize());
}
public int getPieceSize(int index)
{
if (index != networkFile.getPieceCount()-1)
return Math.toIntExact(networkFile.getPieceSize());
int ret = Math.toIntExact(networkFile.getSize() % networkFile.getPieceSize());
if (ret == 0)
ret = Math.toIntExact(networkFile.getPieceSize());
return ret;
}
public byte[] readPiece(int index) throws IOException
{
if (index >= networkFile.getPieceCount())
throw new IllegalArgumentException("Piece index out of range!!");
if (index < 0)
throw new IllegalArgumentException("Piece index is negative!!");
if (!networkFile.hasPiece(index))
return null;
int pieceSize = getPieceSize(index);
byte[] buffer = new byte[pieceSize];
int pieceOffset = getPieceOffset(index);
System.out.println("Reading piece " + index + " from file " + file.getName() + " (offset=" + pieceOffset + ", size=" + pieceSize + ")");
randomAccessFile.seek(pieceOffset);
randomAccessFile.read(buffer);
return buffer;
}
public void writePiece(int index, byte[] buffer) throws IOException
{
if (buffer.length != getPieceSize(index))
throw new IllegalArgumentException("Received a file piece that's the wrong size!! Length = " + buffer.length + " != Piece Size = " + getPieceSize(index));
else if (index >= networkFile.getPieceCount())
throw new IllegalArgumentException("Received a file piece with an index past the end of the file!!");
randomAccessFile.seek(getPieceOffset(index));
randomAccessFile.write(buffer);
networkFile.setHasPiece(index, true);
}
@Override
public void close() throws IOException
{
randomAccessFile.close();
}
}

85
src/main/java/moe/nekojimi/friendcloud/FileRemoteAccess.java Normal file
View file

@ -0,0 +1,85 @@
package moe.nekojimi.friendcloud;
import moe.nekojimi.friendcloud.objects.NetworkFile;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.FutureTask;
public class FileRemoteAccess
{
private final NetworkFile file;
private static final double PREEMPTIVE_DOWNLOAD_THRESHOLD = 5;
// private FilePieceAccess access;
public FileRemoteAccess(NetworkFile file)
{
this.file = file;
}
public synchronized byte[] read(long offset, long size) throws IOException
{
if (offset >= file.getSize())
return new byte[0];
List<Integer> missingPieces = new ArrayList<>();
long pieceSize = file.getPieceSize();
int startPieceIdx = Math.toIntExact(Math.floorDiv(offset, pieceSize));
long endOffset = (offset + size);
if (endOffset >= file.getSize())
endOffset = file.getSize();
int endPieceIdx = Math.toIntExact(Math.floorDiv(endOffset - 1, pieceSize));
for (int pieceIdx = startPieceIdx; pieceIdx <= endPieceIdx; pieceIdx++)
{
if (!file.hasPiece(pieceIdx))
missingPieces.add(pieceIdx);
}
System.out.println("FRA: offset=" + offset + ", endOffset=" + endOffset + ", startPieceIdx=" + startPieceIdx + ", endPieceIdx=" + endPieceIdx);
if (!missingPieces.isEmpty())
{
System.out.println("FRA: need to get missing pieces " + missingPieces);
FileDownloadTask downloadTask = new FileDownloadTask(file, Main.getInstance().getConnectionManager(), missingPieces);
FutureTask<File> futureTask = new FutureTask<>(downloadTask);
Main.getInstance().getExecutor().submit(futureTask);
try
{
futureTask.get();
} catch (InterruptedException | ExecutionException e)
{
e.printStackTrace(System.err);
return null;
}
}
// if (file.getDownloadPercentage() >= PREEMPTIVE_DOWNLOAD_THRESHOLD)
// {
// FileDownloadTask preemptiveDownloadTask = new FileDownloadTask(file, Main.getInstance().getConnectionManager(), )
// }
File localFile = file.getLocalFile();
if (localFile == null)
{
System.err.println("FRA: No local file, read failed!");
return null;
}
long readSize = endOffset - offset;
byte[] ret;
try (RandomAccessFile randomAccessFile = new RandomAccessFile(localFile, "r"))
{
ret = new byte[Math.toIntExact(readSize)];
randomAccessFile.seek(offset);
randomAccessFile.read(ret);
}
return ret;
}
}

286
src/main/java/moe/nekojimi/friendcloud/Main.java Normal file
View file

@ -0,0 +1,286 @@
package moe.nekojimi.friendcloud;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import com.kstruct.gethostname4j.Hostname;
import com.offbynull.portmapper.PortMapperFactory;
import com.offbynull.portmapper.gateway.Bus;
import com.offbynull.portmapper.gateway.Gateway;
import com.offbynull.portmapper.gateways.network.NetworkGateway;
import com.offbynull.portmapper.gateways.process.ProcessGateway;
import com.offbynull.portmapper.mapper.MappedPort;
import com.offbynull.portmapper.mapper.PortMapper;
import com.offbynull.portmapper.mapper.PortType;
import jnr.ffi.Platform;
import moe.nekojimi.friendcloud.filesystem.FUSEAccess;
import moe.nekojimi.friendcloud.network.PeerConnection;
import moe.nekojimi.friendcloud.network.requests.ObjectListRequest;
import moe.nekojimi.friendcloud.objects.NetworkFile;
import moe.nekojimi.friendcloud.objects.NetworkObject;
import moe.nekojimi.friendcloud.objects.Peer;
import moe.nekojimi.friendcloud.objects.PeerFileState;
import moe.nekojimi.friendcloud.protos.ObjectStatements;
import org.slf4j.simple.SimpleLogger;
import org.xml.sax.SAXException;
import javax.xml.parsers.ParserConfigurationException;
import java.io.File;
import java.io.IOException;
import java.net.*;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.concurrent.*;
public class Main
{
private static Main instance;
@Parameter(names="-share")
private List<String> sharedFiles = new ArrayList<>();
@Parameter(names="-known-peer")
private List<String> knownPeers = new ArrayList<>();
@Parameter(names="-tcp-port")
private int tcpPort = 7777;
@Parameter(names="-no-upnp")
private boolean noUpnp = false;
// @Parameter(names="-file")
private ConnectionManager connectionManager;
private final ScheduledExecutorService executor = new ScheduledThreadPoolExecutor(16);
private final FUSEAccess fuseAccess = new FUSEAccess();
public static void main(String[] args)
{
instance = new Main();
JCommander.newBuilder().addObject(instance).build().parse(args);
System.setProperty(SimpleLogger.DEFAULT_LOG_LEVEL_KEY, "Info");
try
{
instance.run();
} catch (IOException | InterruptedException e)
{
e.printStackTrace(System.err);
try
{
instance.shutdown();
} catch (IOException f)
{
throw new RuntimeException(f);
}
System.exit(1);
}
// TestMessage.SearchRequest request = TestMessage.SearchRequest.newBuilder().setQuery("bees!").setPageNumber(316).setResultsPerPage(42069).build();
}
private void run() throws IOException, InterruptedException
{
connectionManager = new ConnectionManager(tcpPort);
Path mountPoint;
if (Platform.getNativePlatform().getOS() == Platform.OS.WINDOWS)
{
mountPoint = Paths.get("J:\\");
}
else
{
mountPoint = Path.of(System.getProperty("user.dir") + "/fuse-mount-" + tcpPort);
boolean created = mountPoint.toFile().mkdirs();
System.out.println("Created FUSE mount point " + mountPoint);
}
fuseAccess.mount(mountPoint);
System.out.println("Mounted virtual filesystem at " + mountPoint);
Runtime.getRuntime().addShutdownHook(new Thread(() ->
{
try
{
shutdown();
} catch (IOException e)
{
throw new RuntimeException(e);
}
}));
connectionManager.addNewConnectionConsumer(this::resquestCompleteState);
connectionManager.start();
String hostname = Hostname.getHostname();
Model.getInstance().getSelfNode().setSystemName(hostname);
Model.getInstance().getSelfNode().setUserName(System.getProperty("user.name") + "-" + tcpPort);
addHostAddress(InetAddress.getLocalHost());
/*
Startup procedure:
- Start up UPnP
- Connect to all known nodes
- Request all changes since most recent
- Apply changes to objects
- Scan local files for changes
- Publish local file changes
*/
if (!noUpnp)
setupIGP();
for (String sharedFilePath: sharedFiles)
{
File file = new File(sharedFilePath);
if (file.exists())
{
System.out.println("Adding shared network file: " + file.getAbsolutePath());
NetworkFile networkFile = (NetworkFile) Model.getInstance().createObjectByType(ObjectStatements.ObjectType.OBJECT_TYPE_FILE);
networkFile.updateFromLocalFile(file);
PeerFileState peerFileState = (PeerFileState) Model.getInstance().createObjectByType(ObjectStatements.ObjectType.OBJECT_TYPE_PEER_FILE_STATE);
peerFileState.setNode(Model.getInstance().getSelfNode());
peerFileState.setFile(networkFile);
peerFileState.setProgress(100);
}
}
for (String knownPeerAddress : knownPeers)
{
String[] split = knownPeerAddress.split(":");
if (split.length != 2)
{
System.err.println("ERROR: " + knownPeerAddress + " isn't a valid address.");
continue;
}
InetSocketAddress address = new InetSocketAddress(split[0],Integer.parseInt(split[1]));
try
{
URI uri = new URI("tcp", null, address.getHostString(), address.getPort(), null, null, null);
PeerConnection nodeConnection = connectionManager.getNodeConnection(uri);
resquestCompleteState(nodeConnection);
// objectListFuture.whenComplete((networkObjects, throwable) -> {
// for (NetworkObject networkObject: networkObjects)
// {
// if (networkObject instanceof NetworkFile)
// {
// System.out.println("Heard about NetworkFile " + networkObject + ", creating download task!");
// FileDownloadTask fileDownloadTask = new FileDownloadTask((NetworkFile) networkObject, connectionManager);
// executor.submit(fileDownloadTask);
// }
// }
// });
} catch (ConnectException ex)
{
System.out.println("Couldn't connect to host " + address);
}
catch (URISyntaxException e)
{
throw new RuntimeException(e);
}
}
}
private void resquestCompleteState(PeerConnection nodeConnection)
{
CompletableFuture<List<NetworkObject>> objectListFuture = nodeConnection.makeRequest(new ObjectListRequest(Set.of(
ObjectStatements.ObjectType.OBJECT_TYPE_FILE,
ObjectStatements.ObjectType.OBJECT_TYPE_PEER_FILE_STATE,
ObjectStatements.ObjectType.OBJECT_TYPE_PEER)));
}
private void addHostAddress(InetAddress address)
{
String host = address.getCanonicalHostName();
Peer selfNode = Model.getInstance().getSelfNode();
try
{
URI uri = new URI("tcp", null, host, tcpPort, null, null, null);
System.out.println("Added local address " + uri);
selfNode.addAddress(uri);
} catch (URISyntaxException e)
{
throw new RuntimeException(e);
}
}
private void setupIGP() throws InterruptedException
{
try
{
// Start gateways
Gateway network = NetworkGateway.create();
Gateway process = ProcessGateway.create();
Bus networkBus = network.getBus();
Bus processBus = process.getBus();
// Discover port forwarding devices and take the first one found
System.out.println("Discovering port mappers...");
List<PortMapper> mappers = PortMapperFactory.discover(networkBus, processBus);;
PortMapper mapper = mappers.getFirst();
System.out.println("Got mapper " + mapper + ", mapping port...");
MappedPort mappedPort = mapper.mapPort(PortType.TCP, tcpPort, tcpPort, 60);
System.out.println("Port mapping added: " + mappedPort);
addHostAddress(mappedPort.getExternalAddress());
}
catch (IllegalStateException ex)
{
System.err.println("Failed to map port! error=" + ex.getMessage());
ex.printStackTrace(System.err);
}
}
private void shutdown() throws IOException
{
fuseAccess.umount();
connectionManager.shutdown();
executor.shutdown();
System.out.println("Waiting 10 seconds to complete tasks...");
boolean terminated = false;
try
{
terminated = executor.awaitTermination(10, TimeUnit.SECONDS);
} catch (InterruptedException e)
{
throw new RuntimeException(e);
}
if (!terminated)
{
System.out.println("Timed out, ending tasks now. Goodbye!");
executor.shutdownNow();
}
else
{
System.out.println("Finished everything. Goodbye!");
}
}
public static Main getInstance()
{
return instance;
}
public ScheduledExecutorService getExecutor()
{
return executor;
}
public ConnectionManager getConnectionManager()
{
return connectionManager;
}
}

139
src/main/java/moe/nekojimi/friendcloud/Model.java Normal file
View file

@ -0,0 +1,139 @@
package moe.nekojimi.friendcloud;
import moe.nekojimi.friendcloud.objects.*;
import moe.nekojimi.friendcloud.protos.ObjectStatements;
import java.util.*;
public class Model
{
private static Model instance = null;
public static Model getInstance()
{
if (instance == null)
instance = new Model();
return instance;
}
private final Map<NetworkObject.ObjectID, NetworkObject> objects = new HashMap<>();
private final int systemID;
private Peer selfPeer = null;
private ObjectChangeRecord changeHead;
private final Map<Long, ObjectChangeRecord> changeRecords = new HashMap<>();
private Model()
{
Random ran = new Random();
systemID = ran.nextInt() & 0x00FFFFFF;
}
public synchronized Peer getSelfNode()
{
if (selfPeer == null)
selfPeer = (Peer) createObjectByType(ObjectStatements.ObjectType.OBJECT_TYPE_PEER);
return selfPeer;
}
// private Map<Long, Node> nodes = new HashMap<>();
public synchronized NetworkObject.ObjectID getNextObjectID(ObjectStatements.ObjectType type)
{
Random ran = new Random();
int randomNumber = ran.nextInt();
NetworkObject.ObjectID objectID = new NetworkObject.ObjectID(type, systemID, randomNumber);
System.out.println("Assigned new object ID: " + objectID);
return objectID;
}
public synchronized NetworkObject createObjectByID(NetworkObject.ObjectID id)
{
ObjectStatements.ObjectType type = id.getType();
System.out.println("Creating new object with type: " + type.name());
NetworkObject ret = switch (type)
{
// case UNRECOGNIZED -> ;
case OBJECT_TYPE_FILE -> new NetworkFile(id);
case OBJECT_TYPE_UNSPECIFIED -> throw new IllegalArgumentException();
// case OBJECT_TYPE_USER -> null;
case OBJECT_TYPE_FOLDER -> new NetworkFolder(id);
case OBJECT_TYPE_PEER -> new Peer(id);
case OBJECT_TYPE_PEER_FILE_STATE -> new PeerFileState(id);
default -> throw new UnsupportedOperationException("NYI");
};
objects.put(id, ret);
return ret;
}
public synchronized NetworkObject createObjectByType(ObjectStatements.ObjectType type)
{
return createObjectByID(getNextObjectID(type));
}
public synchronized NetworkObject getOrCreateObject(NetworkObject.ObjectID id)
{
if (!objects.containsKey(id))
{
objects.put(id, createObjectByID(id));
}
return objects.get(id);
}
public synchronized List<NetworkObject.ObjectID> listObjects(Set<ObjectStatements.ObjectType> types)
{
return objects.keySet().stream().filter((id)->(types.contains(id.getType()))).toList();
}
public synchronized NetworkObject getObject(NetworkObject.ObjectID objectID)
{
return objects.get(objectID);
}
public synchronized List<NetworkFSNode> listFSNodes(String path)
{
//TODO: dumbest algorithm in the world
List<NetworkFSNode> ret = new ArrayList<>();
for (NetworkObject.ObjectID nodeID : listObjects(Set.of(ObjectStatements.ObjectType.OBJECT_TYPE_FILE, ObjectStatements.ObjectType.OBJECT_TYPE_FOLDER)))
{
NetworkFSNode fsNode = (NetworkFSNode) getObject(nodeID);
String networkPath = fsNode.getNetworkPath();
if (networkPath.substring(0, networkPath.lastIndexOf("/")+1).equals(path))
ret.add(fsNode);
}
return ret;
}
public synchronized NetworkFSNode getFSNode(String path)
{
for (NetworkObject.ObjectID nodeID : listObjects(Set.of(ObjectStatements.ObjectType.OBJECT_TYPE_FILE, ObjectStatements.ObjectType.OBJECT_TYPE_FOLDER)))
{
NetworkFSNode fsNode = (NetworkFSNode) getObject(nodeID);
String networkPath = fsNode.getNetworkPath();
if (networkPath.equals(path))
return fsNode;
}
return null;
}
public synchronized void addChangeRecord(ObjectChangeRecord record)
{
changeRecords.put(record.getChangeID(), record);
}
public ObjectChangeRecord getChangeRecord(long id)
{
return changeRecords.get(id);
}
public void applyChangeRecord(ObjectChangeRecord record)
{
if (!record.getChangeHeads().contains(changeHead))
throw new IllegalStateException("Change does not apply! Valid change heads=" + record.getChangeHeads() + ", we are in state " + changeHead.getChangeID());
if (!changeRecords.containsKey(record.getChangeID()))
addChangeRecord(record);
// if (record == null)
// throw new IllegalArgumentException("Cannot apply unknown change!");
}
}

39
src/main/java/moe/nekojimi/friendcloud/ObjectChangeRecord.java Normal file
View file

@ -0,0 +1,39 @@
package moe.nekojimi.friendcloud;
import moe.nekojimi.friendcloud.protos.ObjectStatements;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
public class ObjectChangeRecord
{
private final long changeID;
private ObjectChangeRecord(long changeID)
{
this.changeID = changeID;
}
private final Set<ObjectChangeRecord> changeHeads = new HashSet<>();
public static ObjectChangeRecord createFromChangeMessage(ObjectStatements.ObjectChange objectChange)
{
throw new UnsupportedOperationException("NYI!");
}
public static ObjectChangeRecord createFromObjectStates(ObjectStatements.ObjectState before, ObjectStatements.ObjectState after)
{
throw new UnsupportedOperationException("NYI!");
}
public long getChangeID()
{
return changeID;
}
public Set<ObjectChangeRecord> getChangeHeads()
{
return Collections.unmodifiableSet(changeHeads);
}
}

159
src/main/java/moe/nekojimi/friendcloud/filesystem/FUSEAccess.java Normal file
View file

@ -0,0 +1,159 @@
package moe.nekojimi.friendcloud.filesystem;
import jnr.ffi.Pointer;
import moe.nekojimi.friendcloud.FileRemoteAccess;
import moe.nekojimi.friendcloud.Model;
import moe.nekojimi.friendcloud.objects.NetworkFSNode;
import moe.nekojimi.friendcloud.objects.NetworkFile;
import moe.nekojimi.friendcloud.objects.NetworkFolder;
import ru.serce.jnrfuse.ErrorCodes;
import ru.serce.jnrfuse.FuseFillDir;
import ru.serce.jnrfuse.FuseStubFS;
import ru.serce.jnrfuse.struct.FileStat;
import ru.serce.jnrfuse.struct.FuseFileInfo;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
public class FUSEAccess extends FuseStubFS
{
private static final int DIRECTORY_PERMISSIONS = 0755;
private static final int FILE_PERMISSIONS = 0444;
private final Object fileLock = new Object();
private final Map<Long, FileRemoteAccess> fileAccessors = new HashMap<>();
private final Map<Long, Integer> fileOpenCounts = new HashMap<>();
@Override
public int readdir(String path, Pointer buf, FuseFillDir filter, long offset, FuseFileInfo fi)
{
System.out.println("FUSE: listing contents of directory " + path);
int ret = 0;
filter.apply(buf, ".", null, 0);
filter.apply(buf, "..", null, 0);
// filter.apply(buf,"hello", null, 0);
for (NetworkFSNode fsNode : Model.getInstance().listFSNodes(path))
{
filter.apply(buf, fsNode.getName(), null, 0);
}
return ret;
}
@Override
public int getattr(String path, FileStat stat)
{
// System.out.println("FUSE: reading attributes of file " + path);
if (path.equals("/"))
{
// root directory is special
stat.st_mode.set(FileStat.S_IFDIR | DIRECTORY_PERMISSIONS);
stat.st_nlink.set(2);
}
else
{
NetworkFSNode fsNode = Model.getInstance().getFSNode(path);
switch (fsNode)
{
case null ->
{
return -ErrorCodes.ENOENT();
}
case NetworkFile networkFile ->
{
stat.st_mode.set(FileStat.S_IFREG | FILE_PERMISSIONS);
stat.st_nlink.set(1);
stat.st_size.set(networkFile.getSize());
}
case NetworkFolder networkFolder ->
{
stat.st_mode.set(FileStat.S_IFDIR | DIRECTORY_PERMISSIONS);
stat.st_nlink.set(2);
}
default ->
{
return -ErrorCodes.EIO();
}
}
}
return super.getattr(path, stat);
}
@Override
public int open(String path, FuseFileInfo fi)
{
System.out.println("FUSE: Opening file " + path);
NetworkFSNode fsNode = Model.getInstance().getFSNode(path);
if (fsNode == null)
{
System.err.println("FUSE: Failed to open file " + path + ": not found");
return -ErrorCodes.ENOENT();
}
if (fsNode instanceof NetworkFolder)
{
System.err.println("FUSE: Failed to open file " + path + ": is a directory");
return -ErrorCodes.EISDIR();
}
long fileID = fsNode.getObjectID().getId();
synchronized (fileLock)
{
int openCount = fileOpenCounts.getOrDefault(fileID, 0) + 1;
System.out.println("FUSE: fh: " + fileID + " openCount:" + openCount);
fileOpenCounts.put(fileID, openCount);
if (!fileAccessors.containsKey(fileID))
fileAccessors.put(fileID, new FileRemoteAccess((NetworkFile) fsNode));
}
fi.fh.set(fileID);
return 0;
}
@Override
public int release(String path, FuseFileInfo fi)
{
long fh = fi.fh.longValue();
synchronized (fileLock)
{
Integer openCount = fileOpenCounts.getOrDefault(fh, 1);
openCount -= 1;
System.out.println("FUSE: releasing file " + path + "(fh:" + fh + ", openCount: " + openCount + ")");
fileOpenCounts.put(fh, openCount);
if (openCount <= 0)
{
fileAccessors.remove(fh);
}
}
return 0;
}
@Override
public int read(String path, Pointer buf, long size, long offset, FuseFileInfo fi)
{
System.out.println("FUSE: Reading from file " + path + ", offset=" + offset + ", size=" + size);
synchronized (fileLock)
{
FileRemoteAccess fileRemoteAccess = fileAccessors.get(fi.fh.longValue());
// System.out.println("Got FRA");
byte[] bytes = null;
try
{
bytes = fileRemoteAccess.read(offset, size);
// System.out.println("Read from FRA");
} catch (Exception e)
{
e.printStackTrace(System.err);
return -ErrorCodes.EIO();
}
if (bytes == null)
{
System.err.println("FUSE: failed to read: buffer empty");
return -ErrorCodes.EIO();
}
buf.put(0, bytes, 0, bytes.length);
// System.out.println("FUSE: Read " + bytes.length + " bytes.");
}
return Math.toIntExact(size);
}
}

258
src/main/java/moe/nekojimi/friendcloud/network/PeerConnection.java Normal file
View file

@ -0,0 +1,258 @@
package moe.nekojimi.friendcloud.network;
import com.google.protobuf.Any;
import com.google.protobuf.ByteString;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.Message;
import moe.nekojimi.friendcloud.FilePieceAccess;
import moe.nekojimi.friendcloud.Model;
import moe.nekojimi.friendcloud.objects.NetworkFile;
import moe.nekojimi.friendcloud.objects.NetworkObject;
import moe.nekojimi.friendcloud.objects.Peer;
import moe.nekojimi.friendcloud.protos.CommonMessages;
import moe.nekojimi.friendcloud.protos.ObjectStatements;
import moe.nekojimi.friendcloud.network.requests.Request;
import moe.nekojimi.friendcloud.protos.PieceMessages;
import java.io.IOException;
import java.net.URI;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
public abstract class PeerConnection extends Thread
{
private final Map<Long, Request<?, ?>> pendingRequests = new HashMap<>();
private Peer peer;
private long nextMessageId = 1;
private final URI uri;
private long artificalDelayMs = 0;
public PeerConnection()
{
this(null);
}
public PeerConnection(URI uri)
{
this.uri = uri;
}
public PeerConnection(URI uri, Peer peer)
{
this(uri);
this.peer = peer;
}
@Override
public void run()
{
super.run();
System.out.println("Peer connection to " + uri + " started.");
}
public synchronized <T> CompletableFuture<T> makeRequest(Request<?, T> request)
{
if (!isAlive())
throw new IllegalStateException("Request made to PeerConnection that isn't running!");
try
{
Message message = request.buildMessage();
CommonMessages.FriendCloudMessage wrappedMessage = wrapMessage(message);
pendingRequests.put(wrappedMessage.getHeader().getMessageId(), request);
sendMessage(wrappedMessage);
return request.getFuture();
} catch (Exception e)
{
System.err.println("Request failed!");
e.printStackTrace(System.err);
return CompletableFuture.failedFuture(e);
}
}
protected abstract void sendMessage(CommonMessages.FriendCloudMessage message) throws IOException;
private CommonMessages.FriendCloudMessage wrapMessage(Message message, CommonMessages.MessageHeader inReplyTo)
{
CommonMessages.MessageHeader.Builder headerBuilder = CommonMessages.MessageHeader.newBuilder()
.setMessageId(nextMessageId)
.setSenderId(Model.getInstance().getSelfNode().getObjectID().getId());
if (inReplyTo != null)
headerBuilder.setReplyToMessageId(inReplyTo.getMessageId());
CommonMessages.FriendCloudMessage ret = CommonMessages.FriendCloudMessage.newBuilder()
.setHeader(headerBuilder)
.setBody(Any.pack(message)).build();
nextMessageId++;
return ret;
}
private CommonMessages.FriendCloudMessage wrapMessage(Message message)
{
return wrapMessage(message, null);
}
private void replyWithError(CommonMessages.Error error, CommonMessages.MessageHeader replyHeader) throws IOException
{
System.err.println("Sending error reply: " + error.name() + " to message ID " + replyHeader.getReplyToMessageId());
CommonMessages.ErrorMessage errorMessage = CommonMessages.ErrorMessage.newBuilder().setError(error).build();
sendMessage(wrapMessage(errorMessage,replyHeader));
}
protected void messageReceived(@org.jetbrains.annotations.NotNull CommonMessages.FriendCloudMessage message)
{
if (artificalDelayMs > 0)
{
try
{
System.err.println("WARNING: artifical lag activated! Waiting " + artificalDelayMs + "ms...");
Thread.sleep(artificalDelayMs);
} catch (InterruptedException e)
{
// well never mind then
}
}
CommonMessages.MessageHeader header = message.getHeader();
NetworkObject.ObjectID senderID = new NetworkObject.ObjectID(header.getSenderId());
peer = (Peer) Model.getInstance().getOrCreateObject(senderID);
Any body = message.getBody();
long replyToMessageId = header.getReplyToMessageId();
System.out.println("Received message! type=" + body.getTypeUrl() + ", ID=" + header.getMessageId() + ", reply_to=" + replyToMessageId );
try
{
try
{
if (replyToMessageId != 0)
{
if (pendingRequests.containsKey(replyToMessageId))
handleReplyMessage(header, body);
else if (body.is(CommonMessages.ErrorMessage.class))
handleErrorToUnsolicitedMessage(header, body.unpack(CommonMessages.ErrorMessage.class));
else
replyWithError(CommonMessages.Error.ERROR_NOT_EXPECTING_REPLY, header);
}
else
{
handleUnsolicitedMessage(header, body);
}
}
catch (ReplyWithErrorException ex)
{
ex.printStackTrace(System.err);
replyWithError(ex.getError(), header);
}
catch (IllegalArgumentException ex)
{
ex.printStackTrace(System.err);
replyWithError(CommonMessages.Error.ERROR_INVALID_ARGUMENT, header);
}
// catch (RuntimeException ex)
// {
// ex.printStackTrace(System.err);
// replyWithError(CommonMessages.Error.ERROR_INTERNAL, header);
// }
} catch (IOException ex)
{
ex.printStackTrace(System.err);
}
}
private void handleErrorToUnsolicitedMessage(CommonMessages.MessageHeader header, CommonMessages.ErrorMessage body)
{
throw new RuntimeException("Our message ID " + header.getReplyToMessageId() + " caused a remote error: " + body.getError().name());
}
private void handleUnsolicitedMessage(CommonMessages.MessageHeader header, Any body) throws IOException, ReplyWithErrorException
{
if (body.is(ObjectStatements.ObjectListRequest.class))
{
ObjectStatements.ObjectListRequest objectListRequest = body.unpack(ObjectStatements.ObjectListRequest.class);
List<NetworkObject.ObjectID> objectIDS = Model.getInstance().listObjects(new HashSet<>(objectListRequest.getTypesList()));
ObjectStatements.ObjectList.Builder objectList = ObjectStatements.ObjectList.newBuilder();
for (NetworkObject.ObjectID objectID : objectIDS)
{
NetworkObject networkObject = Model.getInstance().getOrCreateObject(objectID);
objectList.addStates(networkObject.buildObjectState());
// networkObject.updateFromStateMessage();
// objectList.addState(networkObject.buildObjectState());
}
System.out.println("Replying to ObjectListRequest with ObjectList, objects=" + objectList.getStatesList());
sendMessage(wrapMessage(objectList.build(), header));
}
else if (body.is(PieceMessages.FilePiecesRequestMessage.class))
{
PieceMessages.FilePiecesRequestMessage filePiecesRequestMessage = body.unpack(PieceMessages.FilePiecesRequestMessage.class);
if (filePiecesRequestMessage.getPieceMod() == 0)
{
replyWithError(CommonMessages.Error.ERROR_INVALID_ARGUMENT, header);
}
NetworkFile networkFile = (NetworkFile) Model.getInstance().getObject(new NetworkObject.ObjectID(filePiecesRequestMessage.getFileId()));
if (networkFile == null)
{
replyWithError(CommonMessages.Error.ERROR_OBJECT_NOT_FOUND, header);
}
assert networkFile != null;
try (FilePieceAccess filePieceAccess = new FilePieceAccess(networkFile))
{
int startIndex = filePiecesRequestMessage.getStartPieceIndex();
int endIndex = (filePiecesRequestMessage.getStartPieceIndex() + filePiecesRequestMessage.getPieceCount()) - 1;
System.out.println("Been asked for pieces from " + startIndex + " to " + endIndex);
for (int index = startIndex; index <= endIndex; index += filePiecesRequestMessage.getPieceMod())
{
byte[] buffer = filePieceAccess.readPiece(index);
if (buffer != null)
{
System.out.println("Replying to file piece request with piece " + index);
PieceMessages.FilePieceMessage filePieceMessage = PieceMessages.FilePieceMessage.newBuilder()
.setPieceIndex(index)
.setFileId(networkFile.getObjectID().getId())
.setData(ByteString.copyFrom(buffer))
.build();
sendMessage(wrapMessage(filePieceMessage, header));
}
else
{
System.err.println("Don't have requested piece " + index + "!");
replyWithError(CommonMessages.Error.ERROR_PIECE_NOT_POSSESSED, header);
break;
}
}
}
}
}
private void handleReplyMessage(CommonMessages.MessageHeader header, Any body) throws InvalidProtocolBufferException, ReplyWithErrorException
{
long replyToMessageId = header.getReplyToMessageId();
System.out.println("Received reply to message ID " + replyToMessageId);
Request<?, ?> request = pendingRequests.get(replyToMessageId);
boolean doneWithRequest = request.handleReply(body);
if (doneWithRequest)
pendingRequests.remove(replyToMessageId);
}
public abstract void shutdown() throws IOException;
public synchronized Peer getNode()
{
return peer;
}
public synchronized URI getUri()
{
return uri;
}
}

70
src/main/java/moe/nekojimi/friendcloud/network/PeerTCPConnection.java Normal file
View file

@ -0,0 +1,70 @@
package moe.nekojimi.friendcloud.network;
import moe.nekojimi.friendcloud.objects.Peer;
import moe.nekojimi.friendcloud.protos.CommonMessages;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.Socket;
import java.net.URI;
public class PeerTCPConnection extends PeerConnection
{
private final Socket socket;
private final int keepAliveTimeS = 300;
public PeerTCPConnection(URI tcpURL, Peer peer) throws IOException
{
super(tcpURL, peer);
socket = new Socket(tcpURL.getHost(), tcpURL.getPort());
System.out.println("TCP Connection: connected to " + tcpURL + " OK!");
}
public PeerTCPConnection(Socket openSocket)
{
super();
socket = openSocket;
}
@Override
public void run()
{
super.run();
try
{
InputStream inputStream = socket.getInputStream();
while (!socket.isClosed())
{
CommonMessages.FriendCloudMessage message = CommonMessages.FriendCloudMessage.parseDelimitedFrom(inputStream);
// Any any = Any.parseDelimitedFrom(inputStream);
if (message != null)
{
System.out.println("TCP Connection: read data");
messageReceived(message);
}
}
} catch (IOException ex)
{
// fuck
}
}
@Override
protected void sendMessage(CommonMessages.FriendCloudMessage message) throws IOException
{
OutputStream outputStream = socket.getOutputStream();
System.out.println("Sending message " + message.getHeader().getMessageId());
message.writeDelimitedTo(outputStream);
outputStream.flush();
}
@Override
public synchronized void shutdown() throws IOException
{
socket.close();
}
}

25
src/main/java/moe/nekojimi/friendcloud/network/ReplyWithErrorException.java Normal file
View file

@ -0,0 +1,25 @@
package moe.nekojimi.friendcloud.network;
import moe.nekojimi.friendcloud.protos.CommonMessages;
public class ReplyWithErrorException extends Exception
{
private final CommonMessages.Error error;
public ReplyWithErrorException(String message, CommonMessages.Error error)
{
super(message);
this.error = error;
}
public ReplyWithErrorException(CommonMessages.Error error)
{
super("Message raised error:" + error.name());
this.error = error;
}
public CommonMessages.Error getError()
{
return error;
}
}

83
src/main/java/moe/nekojimi/friendcloud/network/requests/FilePiecesRequest.java Normal file
View file

@ -0,0 +1,83 @@
package moe.nekojimi.friendcloud.network.requests;
import com.google.protobuf.Any;
import com.google.protobuf.InvalidProtocolBufferException;
import moe.nekojimi.friendcloud.FilePieceAccess;
import moe.nekojimi.friendcloud.objects.NetworkFile;
import moe.nekojimi.friendcloud.objects.NetworkObject;
import moe.nekojimi.friendcloud.protos.PieceMessages;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
public class FilePiecesRequest extends Request<PieceMessages.FilePiecesRequestMessage, List<Integer>>
{
private final NetworkFile file;
private final int startPiece;
private final int pieceCount;
private final int pieceMod;
private FilePieceAccess filePieceAccess;
private int expectedPieceCount = 0;
// private List<Long> expectedPieces = new ArrayList<>();
private final List<Integer> receivedPieces = new ArrayList<>();
public FilePiecesRequest(NetworkFile file, int startPiece, int pieceCount, int pieceMod)
{
this.file = file;
this.startPiece = startPiece;
this.pieceCount = pieceCount;
this.pieceMod = pieceMod;
}
@Override
public PieceMessages.FilePiecesRequestMessage buildMessage()
{
expectedPieceCount = Math.toIntExact(pieceCount / pieceMod);
return PieceMessages.FilePiecesRequestMessage.newBuilder()
.setFileId(file.getObjectID().getId())
.setPieceCount(pieceCount)
.setPieceMod(pieceMod)
.setStartPieceIndex(startPiece)
.build();
}
@Override
public boolean handleReply(Any reply) throws InvalidProtocolBufferException
{
if (super.handleReply(reply))
return true;
try
{
if (reply.is(PieceMessages.FilePieceMessage.class))
{
expectedPieceCount--;
PieceMessages.FilePieceMessage filePieceMessage = reply.unpack(PieceMessages.FilePieceMessage.class);
byte[] buffer = filePieceMessage.getData().toByteArray();
int index = Math.toIntExact(filePieceMessage.getPieceIndex());
if (filePieceAccess == null)
filePieceAccess = new FilePieceAccess(file);
filePieceAccess.writePiece((int) index, buffer);
receivedPieces.add(index);
}
} catch (IOException ex)
{
future.completeExceptionally(ex);
return true;
}
if (expectedPieceCount <= 0)
{
future.complete(receivedPieces);
}
return expectedPieceCount == 0;
}
}

8
src/main/java/moe/nekojimi/friendcloud/network/requests/Response.java Normal file
View file

@ -0,0 +1,8 @@
package moe.nekojimi.friendcloud.network.requests;
import java.util.concurrent.CompletableFuture;
public class Response<T> extends CompletableFuture<T>
{
}

59
src/main/java/moe/nekojimi/friendcloud/objects/FilePiece.java Normal file
View file

@ -0,0 +1,59 @@
package moe.nekojimi.friendcloud.objects;
import com.google.protobuf.Message;
import java.io.File;
import java.nio.ByteBuffer;
public class FilePiece
{
private final byte[] hash;
private final long size;
private File localFile;
private long fileOffset;
public FilePiece(byte[] hash, long size, File localFile, long fileOffset)
{
this.hash = hash;
this.size = size;
this.localFile = localFile;
this.fileOffset = fileOffset;
}
public FilePiece(byte[] hash, long size)
{
this.hash = hash;
this.size = size;
}
public File getLocalFile()
{
return localFile;
}
public void setLocalFile(File localFile)
{
this.localFile = localFile;
}
public long getFileOffset()
{
return fileOffset;
}
public void setFileOffset(long fileOffset)
{
this.fileOffset = fileOffset;
}
public byte[] getHash()
{
return hash;
}
public long getSize()
{
return size;
}
}

54
src/main/java/moe/nekojimi/friendcloud/objects/NetworkFSNode.java Normal file
View file

@ -0,0 +1,54 @@
package moe.nekojimi.friendcloud.objects;
import moe.nekojimi.friendcloud.Model;
import moe.nekojimi.friendcloud.protos.ObjectStatements;
public abstract class NetworkFSNode extends NetworkObject
{
// private String path = "";
protected NetworkFolder parent = null;
protected String name = "";
public NetworkFSNode(ObjectID objectID)
{
super(objectID);
}
@Override
public synchronized void updateFromStateMessage(ObjectStatements.ObjectState state)
{
super.updateFromStateMessage(state);
if (state.containsValues("name"))
name = state.getValuesOrThrow("name");
if (state.containsValues("parent"))
{
long parentID = Long.parseLong(state.getValuesOrThrow("parent"));
if (parentID != 0)
parent = (NetworkFolder) Model.getInstance().getOrCreateObject(new ObjectID(parentID));
else
parent = null;
}
}
@Override
public ObjectStatements.ObjectState.Builder buildObjectState()
{
return super.buildObjectState()
.putValues("name", getName());
}
public String getName()
{
return name;
}
public void setName(String name)
{
this.name = name;
}
public String getNetworkPath()
{
return (parent != null ? parent.getNetworkPath() : "") + "/" + name;
}
}

233
src/main/java/moe/nekojimi/friendcloud/objects/NetworkFile.java Normal file
View file

@ -0,0 +1,233 @@
package moe.nekojimi.friendcloud.objects;
import moe.nekojimi.friendcloud.protos.ObjectStatements;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.attribute.FileAttribute;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.BitSet;
import java.util.HashMap;
import java.util.HexFormat;
import java.util.Map;
public class NetworkFile extends NetworkFSNode
{
private static final int MIN_PIECE_SIZE = 0x400; // 1KiB
private static final int MAX_PIECE_SIZE = 0x100000; // 1 MiB
private static final int IDEAL_PIECE_COUNT = 1024;
private long size = 0;
private long pieceSize = 0;
private byte[] hash = {};
private File localFile = null;
private final Map<Peer, PeerFileState> fileStates = new HashMap<>();
private BitSet pieces = new BitSet();
private static File tempDirectory = null;
// private List<FilePiece> pieces = new ArrayList<>();
public NetworkFile(ObjectID objectID)
{
super(objectID);
}
public void updateFromLocalFile(File localFile) throws IOException
{
this.localFile = localFile;
name = localFile.getName();
size = localFile.length();
pieceSize = MIN_PIECE_SIZE;
for (pieceSize = MIN_PIECE_SIZE; pieceSize < MAX_PIECE_SIZE; pieceSize *= 2)
{
long pieceCount = size / pieceSize;
if (pieceCount <= IDEAL_PIECE_COUNT)
break;
}
pieces = new BitSet(Math.toIntExact(getPieceCount()));
long offset = 0L;
System.out.println("Calculating hashes for file " + localFile.getName() + "(Piece size: " + pieceSize + ")");
try (FileInputStream input = new FileInputStream(localFile))
{
MessageDigest totalDigest = MessageDigest.getInstance("SHA-256");
byte[] pieceBuf = new byte[Math.toIntExact(pieceSize)];
int pieceIdx = 0;
// List<FilePiece> pieces = new ArrayList<>();
while(true)
{
int bytesRead = input.read(pieceBuf);
if (bytesRead <= 0)
break;
// check to see if this piece is just zeroes, if so, assume it's a missing piece
boolean allZero = true;
for (byte b: pieceBuf)
{
if (b != 0)
{
allZero = false;
break;
}
}
pieces.set(pieceIdx, !allZero);
// MessageDigest pieceDigest = MessageDigest.getInstance("SHA-256");
// pieceDigest.update(pieceBuf, 0, bytesRead);
// byte[] pieceHash = pieceDigest.digest();
totalDigest.update(pieceBuf, 0, bytesRead);
// FilePiece piece = new FilePiece(pieceHash, bytesRead, localFile, offset);
// System.out.print(HexFormat.of().formatHex(pieceHash) + ", ");
// pieces.add(piece);
pieceIdx++;
}
System.out.println();
setLocalFile(localFile);
size = localFile.length();
// setPieces(pieces);
setHash(totalDigest.digest());
System.out.println("Total hash: " + HexFormat.of().formatHex(hash));
System.out.println("Have " + pieces.cardinality() + " of " + getPieceCount() + " pieces.");
} catch (NoSuchAlgorithmException e)
{
throw new RuntimeException(e);
}
}
@Override
public void updateFromStateMessage(ObjectStatements.ObjectState state)
{
super.updateFromStateMessage(state);
// if (state.containsValues("path"))
// path = state.getValuesOrThrow("path");
if (state.containsValues("size"))
size = Long.parseLong(state.getValuesOrThrow("size"));
if (state.containsValues("hash"))
hash = HexFormat.of().parseHex(state.getValuesOrThrow("hash"));
if (state.containsValues("pieceSize"))
pieceSize = Long.parseLong(state.getValuesOrThrow("pieceSize"));
}
@Override
public ObjectStatements.ObjectState.Builder buildObjectState()
{
return super.buildObjectState()
// .putValues("path", path)
.putValues("size", Long.toString(size))
.putValues("hash", HexFormat.of().formatHex(hash))
.putValues("pieceSize", Long.toString(pieceSize));
}
public File getLocalFile()
{
return localFile;
}
public File getOrCreateLocalFile() throws IOException
{
if (tempDirectory == null)
{
tempDirectory = Files.createTempDirectory("FriendCloud").toFile();
tempDirectory.mkdirs();
}
if (localFile == null)
{
localFile = new File(tempDirectory, getName());
// localFile = File.createTempFile("FriendCloud", getNetworkPath());
localFile.createNewFile();
}
return localFile;
}
public void setLocalFile(File localFile)
{
this.localFile = localFile;
}
@Override
public String toString()
{
return "NetworkFile{" +
"hash='" + HexFormat.of().formatHex(hash) + '\'' +
", size=" + size +
", name='" + name + '\'' +
"} " + super.toString();
}
public long getPieceSize()
{
return pieceSize;
}
private void setPieceSize(long pieceSize)
{
this.pieceSize = pieceSize;
}
// public List<FilePiece> getPieces()
// {
// return pieces;
// }
// private void setPieces(List<FilePiece> pieces)
// {
// this.pieces = pieces;
// }
public byte[] getHash()
{
return hash;
}
private void setHash(byte[] hash)
{
this.hash = hash;
}
public int getPieceCount()
{
return Math.toIntExact(Math.ceilDiv(size, pieceSize));
}
public boolean hasPiece(int pieceIdx)
{
return pieces.get(pieceIdx);
}
public void setHasPiece(int pieceIdx, boolean has)
{
pieces.set(pieceIdx, has);
}
public double getDownloadPercentage()
{
return (pieces.cardinality() / (double) pieces.size()) * 100.0;
}
public long getSize()
{
return size;
}
void addFileState(PeerFileState peerFileState)
{
fileStates.put(peerFileState.getNode(), peerFileState);
}
public Map<Peer, PeerFileState> getFileStates()
{
return fileStates;
}
}

25
src/main/java/moe/nekojimi/friendcloud/objects/NetworkFolder.java Normal file
View file

@ -0,0 +1,25 @@
package moe.nekojimi.friendcloud.objects;
import moe.nekojimi.friendcloud.protos.ObjectStatements;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.WeakHashMap;
public class NetworkFolder extends NetworkFSNode
{
// private final SortedSet<ObjectID> children = new TreeSet<>((a,b)->Long.compare(a.getId(),b.getId()));
public NetworkFolder(ObjectID objectID)
{
super(objectID);
}
@Override
public ObjectStatements.ObjectState.Builder buildObjectState()
{
return super.buildObjectState().putValues("name", name);
}
}

101
src/main/java/moe/nekojimi/friendcloud/objects/NetworkObject.java Normal file
View file

@ -0,0 +1,101 @@
package moe.nekojimi.friendcloud.objects;
import moe.nekojimi.friendcloud.protos.ObjectStatements;
import java.util.Objects;
public abstract class NetworkObject
{
private final ObjectID objectID;
public NetworkObject(ObjectID objectID)
{
this.objectID = objectID;
}
public ObjectID getObjectID()
{
return objectID;
}
public synchronized void updateFromStateMessage(ObjectStatements.ObjectState state)
{
if (state.getObjectId() != objectID.getId())
throw new IllegalArgumentException("Wrong object!");
}
public synchronized ObjectStatements.ObjectState mergeChanges(ObjectStatements.ObjectState a, ObjectStatements.ObjectState b)
{
return null;
}
public ObjectStatements.ObjectState.Builder buildObjectState()
{
return ObjectStatements.ObjectState.newBuilder()
.setObjectId(objectID.getId());
}
public static class ObjectID
{
private final ObjectStatements.ObjectType type;
private final int systemID;
private final int uniqueID;
public ObjectID(long id)
{
uniqueID = (int)(0x00000000_FFFFFFFFL & id);
systemID = Math.toIntExact((0x00FFFFFF_00000000L & id) >>> 32);
type = ObjectStatements.ObjectType.forNumber(Math.toIntExact(((0xFF000000_00000000L & id) >>> 56)));
}
public ObjectID(ObjectStatements.ObjectType type, int systemID, int uniqueID)
{
this.type = type;
this.systemID = systemID;
this.uniqueID = uniqueID;
}
public long getId()
{
long uniquePart = Integer.toUnsignedLong(uniqueID);
long systemPart = Integer.toUnsignedLong(systemID) << 32;
long typePart = ((long) type.getNumber()) << 56;
return typePart | systemPart | uniquePart;
}
public ObjectStatements.ObjectType getType()
{
return type;
}
public int getSystemID()
{
return systemID;
}
public int getUniqueID()
{
return uniqueID;
}
@Override
public String toString()
{
return "OBJ{" + Long.toHexString(getId()) + "}";
}
@Override
public boolean equals(Object o)
{
if (o == null || getClass() != o.getClass()) return false;
ObjectID objectID = (ObjectID) o;
return systemID == objectID.systemID && uniqueID == objectID.uniqueID && type == objectID.type;
}
@Override
public int hashCode()
{
return Objects.hash(type, systemID, uniqueID);
}
}
}

123
src/main/java/moe/nekojimi/friendcloud/objects/Peer.java Normal file
View file

@ -0,0 +1,123 @@
package moe.nekojimi.friendcloud.objects;
import moe.nekojimi.friendcloud.protos.ObjectStatements;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
public class Peer extends NetworkObject
{
private final List<URI> addresses = new ArrayList<>();
private String userName = "";
private String systemName = "";
private Map<NetworkFile, PeerFileState> fileStates = new HashMap<>();
private volatile int lastTriedAddressIdx = -1;
public Peer(ObjectID objectID)
{
super(objectID);
}
public String getNodeName()
{
return userName + "@" + systemName;
}
@Override
public synchronized void updateFromStateMessage(ObjectStatements.ObjectState state)
{
super.updateFromStateMessage(state);
Map<String,String> values = state.getValuesMap();
if (values.containsKey("userName"))
userName = values.get("userName");
if (values.containsKey("systemName"))
systemName = values.get("systemName");
if (values.containsKey("addresses"))
{
addresses.clear();
String[] split = values.get("addresses").split(",");
for (String s: split)
{
try
{
addresses.add(new URI(s));
} catch (URISyntaxException e)
{
throw new RuntimeException(e);
}
}
}
// if (values.containsKey("files"))
}
@Override
public ObjectStatements.ObjectState.Builder buildObjectState()
{
ObjectStatements.ObjectState.Builder builder = super.buildObjectState();
if (!userName.isEmpty())
builder.putValues("userName", userName);
if (!systemName.isEmpty())
builder.putValues("systemName", systemName);
if (!addresses.isEmpty())
builder.putValues("addresses",
addresses.stream().map(URI::toString).collect(Collectors.joining(","))
);
return builder;
}
public void addAddress(URI address)
{
addresses.add(address);
}
public URI getNextAddress()
{
lastTriedAddressIdx++;
if (lastTriedAddressIdx >= addresses.size())
lastTriedAddressIdx = 0;
return addresses.get(lastTriedAddressIdx);
}
void addFileState(PeerFileState peerFileState)
{
fileStates.put(peerFileState.getFile(), peerFileState);
}
public Map<NetworkFile, PeerFileState> getFileStates()
{
return fileStates;
}
public List<URI> getAddresses()
{
return addresses;
}
public String getUserName()
{
return userName;
}
public void setUserName(String userName)
{
this.userName = userName;
}
public String getSystemName()
{
return systemName;
}
public void setSystemName(String systemName)
{
this.systemName = systemName;
}
}

75
src/main/java/moe/nekojimi/friendcloud/objects/PeerFileState.java Normal file
View file

@ -0,0 +1,75 @@
package moe.nekojimi.friendcloud.objects;
import moe.nekojimi.friendcloud.Model;
import moe.nekojimi.friendcloud.protos.ObjectStatements;
public class PeerFileState extends NetworkObject
{
private Peer peer;
private NetworkFile file;
private double progress = 0;
public PeerFileState(ObjectID objectID)
{
super(objectID);
}
@Override
public void updateFromStateMessage(ObjectStatements.ObjectState state)
{
super.updateFromStateMessage(state);
peer = (Peer) Model.getInstance().getOrCreateObject(new ObjectID(Long.parseLong(state.getValuesOrThrow("peer"))));
file = (NetworkFile) Model.getInstance().getOrCreateObject(new ObjectID(Long.parseLong(state.getValuesOrThrow("file"))));
if (state.containsValues("progress"))
progress = Double.parseDouble(state.getValuesOrThrow("progress"));
peer.addFileState(this);
file.addFileState(this);
}
@Override
public ObjectStatements.ObjectState mergeChanges(ObjectStatements.ObjectState a, ObjectStatements.ObjectState b)
{
return super.mergeChanges(a, b);
}
@Override
public ObjectStatements.ObjectState.Builder buildObjectState()
{
return super.buildObjectState()
.putValues("peer", Long.toString(peer.getObjectID().getId()))
.putValues("file", Long.toString(file.getObjectID().getId()))
.putValues("progress", Double.toString(progress));
}
public void setNode(Peer peer)
{
this.peer = peer;
}
public void setFile(NetworkFile file)
{
this.file = file;
}
public double getProgress()
{
return progress;
}
public void setProgress(double progress)
{
this.progress = progress;
}
public NetworkFile getFile()
{
return file;
}
public Peer getNode()
{
return peer;
}
}

43
src/main/protobuf/CommonMessages.proto Normal file
View file

@ -0,0 +1,43 @@
syntax = "proto3";
package moe.nekojimi.friendcloud.protos;
import "google/protobuf/any.proto";
message MessageHeader {
uint64 sender_id = 1;
uint64 message_id = 2;
uint64 reply_to_message_id = 3;
}
message FriendCloudMessage {
MessageHeader header = 1;
google.protobuf.Any body = 2;
}
message LoginMessage {
}
enum Error {
ERROR_UNSPECIFIED = 0;
ERROR_WHO_THE_FUCK_ARE_YOU = 1; // sender unidentified or unauthenticated
ERROR_PERMISSION_DENIED = 2; // you can't do that
ERROR_OBJECT_NOT_FOUND = 3; // one or more object(s) specified don't exist
ERROR_INTERNAL = 4; // internal error
ERROR_OUT_OF_DATE = 5; // your action is impossible because you have an out-of-date state (in a way that matters)
ERROR_CHECKSUM_FAILURE = 6; // a supplied checksum didn't match the relevant data
ERROR_PIECE_NOT_POSSESSED = 7; // you asked for a file piece I don't have
ERROR_INVALID_ARGUMENT = 8; // an argument specified is outside the expected range
ERROR_NOT_EXPECTING_REPLY = 9; // you sent a reply to a message that I wasn't expecting a reply to
ERROR_INVALID_PROTOBUF = 10; // your message couldn't be decoded at all
}
message ErrorMessage {
Error error = 1;
}
message PingMessage {
}
message PongMessage {
}

49
src/main/protobuf/ObjectStatements.proto Normal file
View file

@ -0,0 +1,49 @@
syntax = "proto3";
package moe.nekojimi.friendcloud.protos;
import "CommonMessages.proto";
enum ObjectType {
OBJECT_TYPE_UNSPECIFIED = 0;
OBJECT_TYPE_USER = 1;
OBJECT_TYPE_PEER = 2;
OBJECT_TYPE_FILE = 3;
OBJECT_TYPE_FOLDER = 4;
OBJECT_TYPE_PEER_FILE_STATE = 5;
}
message ObjectState {
uint64 object_id = 1;
map<string,string> values = 2;
}
message ObjectStateMessage {
repeated uint64 change_heads = 1;
repeated ObjectState states = 2;
}
message ObjectStateRequest {
repeated uint64 object_ids = 1;
repeated string keys = 2;
}
message ObjectChange {
uint64 change_id = 1;
repeated uint64 change_heads = 2;
repeated ObjectState states = 3;
}
message ObjectChangeRequest {
repeated uint64 changes_since = 1;
}
message ObjectList {
uint64 change_heads = 1;
repeated ObjectState states = 2;
}
message ObjectListRequest {
repeated ObjectType types = 1;
}

46
src/main/protobuf/PieceMessages.proto Normal file
View file

@ -0,0 +1,46 @@
syntax = "proto3";
package moe.nekojimi.friendcloud.protos;
import "CommonMessages.proto";
message FilePieceChange {
uint64 file_id = 1;
repeated uint64 changed_index = 2;
}
message FilePieceChangeMessage {
uint64 change_id = 1;
repeated FilePieceChange changes = 2;
}
message FilePieceChangeRequestMessage {
uint64 change_id = 1;
}
message FilePiecesRequestMessage {
uint64 file_id = 1;
uint32 start_piece_index = 2;
uint32 piece_count = 3;
uint32 piece_mod = 4;
}
message FilePiecesQueryMessage {
uint64 file_id = 1;
}
message FilePiecesQueryResultMessage {
bytes bitfield = 1;
}
//message PieceRequestMessage {
// bytes piece_hash = 1;
//}
//
message FilePieceMessage
{
bytes piece_hash = 1;
uint64 file_id = 2;
uint32 piece_index = 3;
bytes data = 4;
}

9
src/main/protobuf/TestMessage.proto Normal file
View file

@ -0,0 +1,9 @@
syntax = "proto3";
option java_package = "moe.nekojimi.friendcloud.protos";
message SearchRequest {
string query = 1;
int32 page_number = 2;
int32 results_per_page = 3;
}