OMGeeky/drive_syncer
1
0
Fork 0
mirror of https://github.com/OMGeeky/drive_syncer.git synced 2025-12-26 16:17:24 +01:00
Code Issues Packages Projects Releases Wiki Activity

streaming works, is slow but works. no writing yet

Browse Source
This commit is contained in:
OMGeeky
2023-05-14 17:20:38 +02:00
commit faf49050e7
17 changed files with 2410 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored Normal file
View File

@@ -0,0 +1,4 @@
/target
/Cargo.lock
/auth
/.idea

26
Cargo.toml Normal file
View File

@@ -0,0 +1,26 @@
[package]
name = "untitled"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
google-drive3 = "5.0"
tokio = { version = "1.28", features = ["full"] }
log = "0.4"
env_logger = "0.10"
#nix = { version = "0.26", features = ["mount"] }
tempfile = "3.5.0"
notify = { version = "5.1", default-features = false, features = ["macos_kqueue"]}
fuser = "0.12"
async-trait = "0.1.68"
libc = "0.2"
reqwest = "0.11.17"
bytes = "1.4.0"
futures = "0.3.28"
hyper = { version = "0.14.24", features = ["full", "stream"] }
mime = "0.3"
anyhow = "1.0"
async-recursion = "1.0.4"

18
src/async_helper.rs Normal file
View File

@@ -0,0 +1,18 @@
use log::{debug, trace};
use std::future::Future;
use tokio::runtime::{Handle, Runtime};
/// Run a future to completion on the current thread.
/// This is useful when you want to run a future in a blocking context.
/// This function will block the current thread until the provided future has run to completion.
///
/// # Be careful with deadlocks
pub fn run_async_blocking<T>(f: impl std::future::Future<Output = T> + Sized) -> T {
trace!("run_async");
let handle = Handle::current();
handle.enter();
trace!("run_async: entered handle");
let result = futures::executor::block_on(f);
trace!("run_async: got result");
result
}

81
src/common.rs Normal file
View File

@@ -0,0 +1,81 @@
use std::ffi::{OsStr, OsString};
use std::ops::Deref;
use std::path::{Path, PathBuf};
#[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq, Hash)]
pub struct LocalPath(PathBuf);
impl From<PathBuf> for LocalPath {
fn from(path: PathBuf) -> Self {
Self(path)
}
}
impl From<&Path> for LocalPath {
fn from(path: &Path) -> Self {
Self(path.to_path_buf())
}
}
impl From<&PathBuf> for LocalPath {
fn from(path: &PathBuf) -> Self {
Self(path.to_path_buf())
}
}
impl From<OsString> for LocalPath {
fn from(path: OsString) -> Self {
Self::from(&path)
}
}
impl From<&OsString> for LocalPath {
fn from(path: &OsString) -> Self {
Path::new(path).into()
}
}
impl<T> AsRef<T> for LocalPath
where
T: ?Sized,
<PathBuf as Deref>::Target: AsRef<T>,
{
fn as_ref(&self) -> &T {
self.0.deref().as_ref()
}
}
impl Deref for LocalPath {
type Target = PathBuf;
fn deref(&self) -> &Self::Target {
&self.0
}
}
//------------------------------------------
impl Into<PathBuf> for LocalPath {
fn into(self) -> PathBuf {
self.0
}
}
impl Into<OsString> for LocalPath {
fn into(self) -> OsString {
self.0.into_os_string()
}
}
impl<'a> Into<&'a Path> for &'a LocalPath {
fn into(self) -> &'a Path {
&self.0
}
}
impl<'a> Into<&'a OsStr> for &'a LocalPath {
fn into(self) -> &'a OsStr {
self.0.as_os_str()
}
}
impl<'a> Into<&'a PathBuf> for &'a LocalPath {
fn into(self) -> &'a PathBuf {
&self.0
}
}

159
src/fs/common.rs Normal file
View File

@@ -0,0 +1,159 @@
use crate::async_helper::run_async_blocking;
use crate::common::LocalPath;
use crate::fs::inode::Inode;
use crate::google_drive::DriveId;
use crate::prelude::*;
use anyhow::anyhow;
use async_trait::async_trait;
use fuser::{FileAttr, FileType, TimeOrNow, FUSE_ROOT_ID};
use log::debug;
use std::collections::HashMap;
use std::ffi::{OsStr, OsString};
use std::path::{Path, PathBuf};
use std::time::SystemTime;
pub trait CommonEntry {
fn get_ino(&self) -> Inode;
fn get_name(&self) -> &OsStr;
fn get_local_path(&self) -> &LocalPath;
fn get_attr(&self) -> &FileAttr;
// fn new(
// ino: impl Into<Inode>,
// name: impl Into<OsString>,
// local_path: impl Into<LocalPath>,
// attr: FileAttr,
// ) -> Self;
}
#[async_trait]
pub trait CommonFilesystem<Entry: CommonEntry> {
fn get_entries(&self) -> &HashMap<Inode, Entry>;
fn get_entries_mut(&mut self) -> &mut HashMap<Inode, Entry>;
fn get_children(&self) -> &HashMap<Inode, Vec<Inode>>;
fn get_children_mut(&mut self) -> &mut HashMap<Inode, Vec<Inode>>;
fn get_root_path(&self) -> LocalPath;
fn generate_ino(&self) -> Inode {
Inode::new(self.get_entries().len() as u64 + 1) //TODO: check if this is working or if concurrency is a problem
}
fn get_path_from_ino(&self, ino: impl Into<Inode>) -> Option<LocalPath> {
let ino = ino.into();
debug!("get_path_from_ino: {}", ino);
let res = self.get_entry(ino)?.get_local_path().clone();
debug!("get_path_from_ino: {}:{:?}", ino, res);
Some(res)
}
fn get_full_path_from_ino(&self, ino: impl Into<Inode>) -> Option<LocalPath> {
let ino = ino.into();
debug!("get_full_path_from_ino: {}", ino);
if ino == FUSE_ROOT_ID.into() {
return Some(self.get_root_path());
}
let parent = self.get_parent_ino(ino);
if let Some(parent) = parent {
let path: PathBuf = self.get_full_path_from_ino(parent)?.into();
let buf: LocalPath = path
.join::<PathBuf>(self.get_path_from_ino(ino)?.into())
.into();
debug!("get_full_path_from_ino: {}:{:?}", ino, buf);
return Some(buf);
}
match self.get_path_from_ino(ino) {
Some(path) => Some(path.clone()),
None => None,
}
}
fn get_child_with_path(
&self,
parent: impl Into<Inode>,
path: impl AsRef<OsStr>,
) -> Option<Inode> {
let parent = parent.into();
let path = path.as_ref();
debug!("get_child_with_path: {}:{:?}", parent, path);
let children = self.get_children().get(&parent)?;
let mut res = None;
for child in children {
let child_path: &OsStr = self.get_entry(*child)?.get_local_path().into();
if child_path == path {
res = Some(*child);
break;
}
}
debug!("get_child_with_path: {}:{:?}", parent, res);
res
}
fn get_parent_ino(&self, ino: impl Into<Inode>) -> Option<Inode> {
let ino = ino.into();
debug!("get_parent_ino: {}", ino);
if ino == FUSE_ROOT_ID.into() {
return None;
}
let mut parent = None;
for (parent_ino, child_inos) in self.get_children().iter() {
if child_inos.contains(&ino) {
parent = Some(*parent_ino);
break;
}
}
parent
}
fn convert_to_system_time(mtime: TimeOrNow) -> SystemTime {
let mtime = match mtime {
TimeOrNow::SpecificTime(t) => t,
TimeOrNow::Now => SystemTime::now(),
};
mtime
}
fn get_entry(&self, ino: impl Into<Inode>) -> Option<&Entry> {
self.get_entries().get(&ino.into())
}
fn get_entry_r(&self, ino: impl Into<Inode>) -> Result<&Entry> {
self.get_entries()
.get(&ino.into())
.ok_or(anyhow!("Entry not found").into())
}
async fn add_file_entry(
&mut self,
parent: impl Into<Inode> + Send,
name: &OsStr,
mode: u16,
size: u64,
) -> Result<Inode> {
let parent = parent.into();
debug!("add_file_entry: {}:{:?}; {}", parent, name, mode);
let ino = self
.add_entry(name, mode, FileType::RegularFile, parent, size)
.await?;
Ok(ino)
}
async fn add_entry(
&mut self,
name: &OsStr,
mode: u16,
file_type: FileType,
parent_ino: impl Into<Inode> + Send,
size: u64,
) -> Result<Inode>;
fn add_child(&mut self, parent_ino: impl Into<Inode>, ino: impl Into<Inode>) {
let parents_child_list = self
.get_children_mut()
.entry(parent_ino.into())
.or_default();
let ino: Inode = ino.into();
if !parents_child_list.contains(&ino) {
parents_child_list.push(ino);
}
}
}

53
src/fs/drive/entry.rs Normal file
View File

@@ -0,0 +1,53 @@
use crate::common::LocalPath;
use crate::fs::{CommonEntry, Inode};
use crate::google_drive::DriveId;
use fuser::FileAttr;
use std::ffi::{OsStr, OsString};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DriveEntry {
pub ino: Inode,
pub drive_id: DriveId,
pub name: OsString,
// pub drive_path: OsString,
pub local_path: LocalPath,
pub attr: FileAttr,
}
impl DriveEntry {
pub fn new(
ino: impl Into<Inode>,
name: impl Into<OsString>,
drive_id: impl Into<DriveId>,
local_path: impl Into<LocalPath>,
attr: FileAttr,
) -> Self {
let name = name.into();
let path = local_path.into();
Self {
ino: ino.into(),
drive_id: drive_id.into(),
name,
// drive_path: path.clone().into(),
local_path: path,
attr,
}
}
}
impl CommonEntry for DriveEntry {
fn get_ino(&self) -> Inode {
self.ino
}
fn get_name(&self) -> &OsStr {
&self.name
}
fn get_local_path(&self) -> &LocalPath {
&self.local_path
}
fn get_attr(&self) -> &FileAttr {
&self.attr
}
}

546
src/fs/drive/mod.rs Normal file
View File

@@ -0,0 +1,546 @@
use crate::common::LocalPath;
use crate::fs::common::CommonFilesystem;
use crate::fs::inode::Inode;
use crate::fs::CommonEntry;
use crate::google_drive::{DriveId, GoogleDrive};
use crate::prelude::*;
use crate::prelude::*;
use anyhow::{anyhow, Error};
use async_recursion::async_recursion;
use drive3::api::File;
use fuser::{
FileAttr, FileType, Filesystem, KernelConfig, ReplyAttr, ReplyData, ReplyDirectory, ReplyEmpty,
ReplyEntry, ReplyOpen, ReplyStatfs, ReplyWrite, ReplyXattr, Request, TimeOrNow, FUSE_ROOT_ID,
};
use futures::TryFutureExt;
use libc::c_int;
use log::{debug, error, warn};
use mime::Mime;
use std::any::Any;
use std::collections::HashMap;
use std::ffi::{OsStr, OsString};
use std::fmt::Display;
use std::fs::OpenOptions;
use std::os::unix::prelude::*;
use std::path::{Path, PathBuf};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use tempfile::TempDir;
use tokio::io::{stdin, AsyncBufReadExt};
use tokio::runtime::Runtime;
mod entry;
use crate::async_helper::run_async_blocking;
pub use entry::*;
#[derive(Debug)]
pub struct DriveFilesystem {
// runtime: Runtime,
/// the point where the filesystem is mounted
root: PathBuf,
/// the source dir to read from and write to
source: GoogleDrive,
/// the cache dir to store the files in
cache_dir: Option<TempDir>,
/// How long the responses can/should be cached
time_to_live: Duration,
entries: HashMap<Inode, DriveEntry>,
children: HashMap<Inode, Vec<Inode>>,
/// The generation of the filesystem
/// This is used to invalidate the cache
/// when the filesystem is remounted
generation: u64,
}
impl DriveFilesystem {
pub async fn new(root: impl AsRef<Path>) -> Result<Self> {
debug!("new: {:?};", root.as_ref());
let mut entries = HashMap::new();
let now = SystemTime::now();
// Add root directory with inode number 1
let root_attr = FileAttr {
ino: FUSE_ROOT_ID,
size: 0,
blocks: 0,
atime: now,
mtime: now,
ctime: now,
crtime: now,
kind: FileType::Directory,
perm: 0o755,
nlink: 2,
uid: 0,
gid: 0,
rdev: 0,
blksize: 4096,
flags: 0,
};
let inode = FUSE_ROOT_ID.into();
entries.insert(
inode,
DriveEntry {
ino: inode,
name: "root".into(),
local_path: LocalPath::from(Path::new("")),
drive_id: DriveId::root(),
// drive_path: "/".into(),
attr: root_attr,
},
);
let cache_dir = tempfile::tempdir()?;
debug!("cache_dir: {:?}", cache_dir.path());
if !cache_dir.path().exists() {
debug!("creating cache dir: {:?}", cache_dir.path());
std::fs::create_dir_all(cache_dir.path())?;
} else {
debug!("cache dir exists: {}", cache_dir.path().display());
}
let mut s = Self {
root: root.as_ref().to_path_buf(),
source: GoogleDrive::new().await?,
cache_dir: Some(cache_dir),
time_to_live: Duration::from_secs(2),
entries,
/*TODO: implement a way to increase this if necessary*/
generation: 0,
children: HashMap::new(),
};
//
// let root = s.root.to_path_buf();
// s.add_dir_entry(&root, Inode::from(FUSE_ROOT_ID), true)
// .await;
Ok(s)
}
fn get_cache_dir_for_file(&self, inode: Inode) -> Result<PathBuf> {
debug!("get_cache_dir_for_file: {}", inode);
let cache_dir = self.cache_dir.as_ref().ok_or(anyhow!("no cache dir"))?;
debug!(
"get_cache_dir_for_file: {}, cache_dir: {}",
inode,
cache_dir.path().display()
);
let entry = self
.entries
.get(&inode)
.ok_or(anyhow!("could not get entry"))?;
debug!(
"get_cache_dir_for_file: entry local_path: {}",
entry.local_path.display()
);
let folder_path = match entry.local_path.parent() {
Some(p) => p.as_os_str(),
None => OsStr::new(""),
};
debug!("get_cache_dir_for_file: folder_path: {:?}", folder_path);
let path = cache_dir.path().join(folder_path);
debug!("get_cache_dir_for_file: {}: {}", inode, path.display());
Ok(path)
}
#[async_recursion::async_recursion]
async fn add_dir_entry(
&mut self,
folder_path: &Path,
parent_ino: impl Into<Inode> + Send + 'async_recursion,
skip_self: bool,
) -> Result<()> {
let parent_ino: Inode = parent_ino.into();
let ino;
debug!(
"add_dir_entry: {:?}; parent: {}; skip_self: {} ",
folder_path, parent_ino, skip_self
);
if self.root == folder_path {
ino = parent_ino;
} else {
debug!("add_dir_entry: adding entry for {:?}", folder_path);
ino = self
.add_entry(
folder_path.file_name().ok_or(anyhow!("invalid filename"))?,
/*TODO: correct permissions*/
0o755,
FileType::Directory,
parent_ino,
/*TODO: implement size for folders*/ 0,
)
.await?;
}
let drive = &self.source;
let folder_drive_id: DriveId = self
.get_drive_id(ino)
.ok_or(anyhow!("could not find dir drive_id"))?;
debug!(
"add_dir_entry: getting files for '{:50?}' {}",
folder_drive_id,
folder_path.display()
);
let files;
{
let files_res = self.source.list_files(folder_drive_id).await;
if let Err(e) = files_res {
warn!("could not get files: {}", e);
return Ok(());
}
files = files_res.unwrap();
}
debug!("got {} files", files.len());
// let d = std::fs::read_dir(folder_path);
for entry in files {
debug!("entry: {:?}", entry);
let name = entry.name.as_ref().ok_or_else(|| "no name");
if let Err(e) = name {
warn!("could not get name: {}", e);
continue;
}
let name = name.as_ref().unwrap();
if name.contains("/") || name.contains("\\") || name.contains(":") {
warn!("invalid name: {}", name);
continue;
}
let path = folder_path.join(&name);
if let None = &entry.mime_type {
warn!("could not get mime_type");
continue;
}
let mime_type = entry.mime_type.as_ref().unwrap();
if mime_type == "application/vnd.google-apps.document"
|| mime_type == "application/vnd.google-apps.spreadsheet"
|| mime_type == "application/vnd.google-apps.drawing"
|| mime_type == "application/vnd.google-apps.form"
|| mime_type == "application/vnd.google-apps.presentation"
|| mime_type == "application/vnd.google-apps.drive-sdk"
|| mime_type == "application/vnd.google-apps.script"
//TODO: add all relevant mime types
{
debug!(
"skipping google file: mime_type: '{}' entry: {:?}",
mime_type, entry
);
continue;
} else if mime_type == "application/vnd.google-apps.folder" {
debug!("adding folder: {:?}", path);
let res = self.add_dir_entry(&path, ino, false).await;
if let Err(e) = res {
warn!("could not add folder: {}", e);
continue;
}
// } else if metadata.is_file() {
} else {
debug!("adding file: '{}' {:?}", mime_type, path);
let size = match Self::get_size_from_drive_metadata(&entry) {
Some(value) => value,
None => continue,
};
let mode = 0o644; //TODO: get mode from settings
self.add_file_entry(ino, &OsString::from(&name), mode as u16, size)
.await;
}
}
Ok(())
}
fn get_size_from_drive_metadata(entry: &File) -> Option<u64> {
let size = entry.size.ok_or_else(|| 0);
if let Err(e) = size {
warn!("could not get size: {}", e);
return None;
}
let size = size.unwrap();
if size < 0 {
warn!("invalid size: {}", size);
return None;
}
let size = size as u64;
Some(size)
}
fn get_drive_id(&self, ino: impl Into<Inode>) -> Option<DriveId> {
self.get_entry(ino).map(|e| e.drive_id.clone())
}
async fn download_file_to_cache(&self, ino: impl Into<Inode>) -> Result<PathBuf> {
let ino = ino.into();
debug!("download_file_to_cache: {}", ino);
let entry = self.get_entry_r(ino)?;
let drive_id = entry.drive_id.clone();
let drive = &self.source;
let path = self.get_cache_path_for_entry(&entry)?;
let folder = path.parent().unwrap();
if !folder.exists() {
debug!("creating folder: {}", folder.display());
std::fs::create_dir_all(folder)?;
}
debug!("downloading file: {}", path.display());
drive.download_file(drive_id, &path).await?;
Ok(path)
}
fn check_if_file_is_cached(&self, ino: impl Into<Inode>) -> Result<bool> {
let entry = self.get_entry_r(ino)?;
let path = self.get_cache_path_for_entry(&entry)?;
let exists = path.exists();
Ok(exists)
}
fn get_cache_path_for_entry(&self, entry: &&DriveEntry) -> Result<PathBuf> {
debug!("get_cache_path_for_entry: {}", entry.ino);
let folder = self.get_cache_dir_for_file(entry.ino)?;
let path = folder.join(&entry.name);
debug!(
"get_cache_path_for_entry: {}: {}",
entry.ino,
path.display()
);
Ok(path)
}
}
#[async_trait::async_trait]
impl CommonFilesystem<DriveEntry> for DriveFilesystem {
fn get_entries(&self) -> &HashMap<Inode, DriveEntry> {
&self.entries
}
fn get_entries_mut(&mut self) -> &mut HashMap<Inode, DriveEntry> {
&mut self.entries
}
fn get_children(&self) -> &HashMap<Inode, Vec<Inode>> {
&self.children
}
fn get_children_mut(&mut self) -> &mut HashMap<Inode, Vec<Inode>> {
&mut self.children
}
fn get_root_path(&self) -> LocalPath {
self.root.clone().into()
}
async fn add_entry(
&mut self,
name: &OsStr,
mode: u16,
file_type: FileType,
parent_ino: impl Into<Inode> + Send,
size: u64,
) -> Result<Inode> {
let parent_ino = parent_ino.into();
debug!("add_entry: (0) name:{:20?}; parent: {}", name, parent_ino);
let ino = self.generate_ino(); // Generate a new inode number
let now = std::time::SystemTime::now();
let attr = FileAttr {
ino: ino.into(),
size: size,
blocks: 0,
atime: now,
mtime: now,
ctime: now,
crtime: now,
kind: file_type,
perm: mode,
nlink: 1,
uid: 0,
gid: 0,
rdev: 0,
blksize: 4096,
flags: 0,
};
let parent_drive_id = self.get_drive_id(parent_ino);
let drive_id: DriveId = self.source.get_id(name, parent_drive_id).await?;
debug!("add_entry: (1) drive_id: {:?}", drive_id);
let parent_local_path = self.get_path_from_ino(parent_ino);
let parent_path: PathBuf = parent_local_path
.ok_or(anyhow!("could not get local path"))?
.into();
self.get_entries_mut().insert(
ino,
DriveEntry::new(ino, name, drive_id, parent_path.join(name), attr),
);
self.add_child(parent_ino, &ino);
debug!("add_entry: (2) after adding count: {}", self.entries.len());
Ok(ino)
}
}
impl Display for DriveFilesystem {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "DriveFilesystem at: '/{}'", self.root.display())
}
}
impl Filesystem for DriveFilesystem {
fn init(
&mut self,
_req: &Request<'_>,
_config: &mut KernelConfig,
) -> std::result::Result<(), c_int> {
debug!("init");
let root = self.root.to_path_buf();
let x = run_async_blocking(self.add_dir_entry(&root, Inode::from(FUSE_ROOT_ID), true));
if let Err(e) = x {
error!("could not add root entry: {}", e);
}
Ok(())
}
fn destroy(&mut self) {
debug!("destroy");
debug!("destroy: removing cache dir: {:?}", self.cache_dir);
self.cache_dir = None;
}
fn lookup(&mut self, _req: &Request<'_>, parent: u64, name: &OsStr, reply: ReplyEntry) {
debug!("lookup: {}:{:?}", parent, name);
let parent = parent.into();
let children = self.children.get(&parent);
if children.is_none() {
warn!("lookup: could not find children for {}", parent);
reply.error(libc::ENOENT);
return;
}
let children = children.unwrap();
debug!("lookup: children: {:?}", children);
for child_inode in children {
let entry = self.entries.get(child_inode);
if entry.is_none() {
warn!("lookup: could not find entry for {}", child_inode);
continue;
}
let entry = entry.unwrap();
let path: PathBuf = entry.name.clone().into();
let accepted = name.eq_ignore_ascii_case(&path);
debug!(
"entry: {}:(accepted={}){:?}; {:?}",
child_inode, accepted, path, entry.attr
);
if accepted {
reply.entry(&self.time_to_live, &entry.attr, self.generation);
return;
}
}
warn!("lookup: could not find entry for {:?}", name);
reply.error(libc::ENOENT);
}
fn getattr(&mut self, _req: &Request<'_>, ino: u64, reply: ReplyAttr) {
debug!("getattr: {}", ino);
let entry = self.entries.get(&ino.into());
if let Some(entry) = entry {
reply.attr(&self.time_to_live, &entry.attr);
} else {
reply.error(libc::ENOENT);
}
}
fn read(
&mut self,
_req: &Request<'_>,
ino: u64,
fh: u64,
offset: i64,
size: u32,
flags: i32,
lock_owner: Option<u64>,
reply: ReplyData,
) {
debug!(
"read: {:10}:{:2}:{:3}:{:10}:{:10X}:{:?}",
ino, fh, offset, size, flags, lock_owner
);
let entry = self.get_entry_r(&ino.into());
if let Err(e) = entry {
error!("read: could not find entry for {}: {}", ino, e);
reply.error(libc::ENOENT);
return;
}
let entry = entry.unwrap();
let is_cached = self.check_if_file_is_cached(ino);
if !is_cached.unwrap_or(false) {
debug!("read: file is not cached: {}", ino);
let x: Result<PathBuf> = run_async_blocking(self.download_file_to_cache(ino));
if let Err(e) = x {
error!("read: could not download file: {}", e);
reply.error(libc::ENOENT);
return;
}
}
let path = self.get_cache_path_for_entry(&entry);
if let Err(e) = path {
error!("read: could not get cache path: {}", e);
reply.error(libc::ENOENT);
return;
}
let path = path.unwrap();
debug!("read: path: {:?}", path);
let file = std::fs::File::open(&path);
if let Err(e) = file {
error!("read: could not open file: {}", e);
reply.error(libc::ENOENT);
return;
}
let mut file = file.unwrap();
let mut buf = vec![0; size as usize];
debug!("reading file: {:?} at {} with size {}", &path, offset, size);
file.read_at(&mut buf, offset as u64).unwrap();
debug!("read file: {:?} at {}", &path, offset);
reply.data(&buf);
}
fn readdir(
&mut self,
_req: &Request<'_>,
ino: u64,
fh: u64,
mut offset: i64,
mut reply: ReplyDirectory,
) {
debug!("readdir: {}:{}:{:?}", ino, fh, offset);
let children = self.children.get(&ino.into());
if let Some(attr) = self.get_entries().get(&ino.into()).map(|entry| entry.attr) {
if attr.kind != FileType::Directory {
reply.error(libc::ENOTDIR);
return;
}
}
if !(children.is_none()) {
} else {
reply.error(libc::ENOENT);
return;
}
let children = children.unwrap();
debug!("children ({}): {:?}", children.len(), children);
for child_inode in children.iter().skip(offset as usize) {
let entry = self.entries.get(child_inode).unwrap();
let path: PathBuf = entry.local_path.clone().into();
let attr = entry.attr;
let inode = (*child_inode).into();
offset += 1; // Increment the offset for each processed entry
debug!("entry: {}:{:?}; {:?}", inode, path, attr);
if reply.add(inode, offset, attr.kind, &entry.name) {
// If the buffer is full, we need to stop
debug!("readdir: buffer full");
break;
}
}
debug!("readdir: ok");
reply.ok();
}
fn access(&mut self, _req: &Request<'_>, ino: u64, mask: i32, reply: ReplyEmpty) {
reply.ok(); //TODO: implement this correctly
}
}

56
src/fs/inode.rs Normal file
View File

@@ -0,0 +1,56 @@
use std::fmt::Display;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct Inode(u64);
impl Inode {
pub fn new(value: u64) -> Self {
Self(value)
}
pub fn get(&self) -> u64 {
self.0
}
}
impl Display for Inode {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.0)
}
}
impl Into<u64> for Inode {
fn into(self) -> u64 {
self.0
}
}
// impl Into<Inode> for Inode{
// fn into(self) -> Inode {
// self
// }
// }
impl TryInto<u32> for Inode {
type Error = std::num::TryFromIntError;
fn try_into(self) -> Result<u32, Self::Error> {
self.0.try_into()
}
}
impl From<u64> for Inode {
fn from(value: u64) -> Inode {
Inode(value)
}
}
impl From<u32> for Inode {
fn from(value: u32) -> Inode {
Inode(value as u64)
}
}
impl From<&Inode> for Inode {
fn from(value: &Inode) -> Self {
value.clone()
}
}

6
src/fs/mod.rs Normal file
View File

@@ -0,0 +1,6 @@
mod common;
mod inode;
pub use common::*;
pub use inode::*;
pub mod drive;
pub mod sample;

474
src/fs/sample.rs Normal file
View File

@@ -0,0 +1,474 @@
// use crate::async_helper::run_async_in_sync;
use crate::async_helper::run_async_blocking;
use crate::common::LocalPath;
use crate::fs::common::CommonFilesystem;
use crate::fs::inode::Inode;
use crate::fs::CommonEntry;
use crate::prelude::*;
use fuser::{
FileAttr, FileType, KernelConfig, ReplyAttr, ReplyData, ReplyDirectory, ReplyEmpty, ReplyEntry,
ReplyOpen, ReplyStatfs, ReplyWrite, ReplyXattr, Request, TimeOrNow, FUSE_ROOT_ID,
};
use libc::c_int;
use log::{debug, warn};
use std::collections::HashMap;
use std::ffi::{OsStr, OsString};
use std::fmt::Display;
use std::fs::OpenOptions;
use std::os::unix::prelude::*;
use std::path::{Path, PathBuf};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
#[derive(Debug)]
struct SampleEntry {
pub ino: Inode,
pub name: OsString,
pub local_path: LocalPath,
pub attr: FileAttr,
}
impl SampleEntry {
// fn new(ino: impl Into<Inode>, local_path: OsString, attr: FileAttr) -> Self {
// Self {
// ino: ino.into(),
// name: OsString::new(),
// local_path: LocalPath::from(Path::new(&local_path)),
// attr,
// }
// }
fn new(
ino: impl Into<Inode>,
name: impl Into<OsString>,
local_path: impl Into<LocalPath>,
attr: FileAttr,
) -> Self {
Self {
ino: ino.into(),
name: name.into(),
local_path: local_path.into(),
attr,
}
}
}
impl CommonEntry for SampleEntry {
fn get_ino(&self) -> Inode {
self.ino
}
fn get_name(&self) -> &OsStr {
self.name.as_os_str()
}
fn get_local_path(&self) -> &LocalPath {
&self.local_path
}
fn get_attr(&self) -> &FileAttr {
&self.attr
}
}
#[derive(Debug, Default)]
pub struct SampleFilesystem {
/// the point where the filesystem is mounted
root: PathBuf,
/// the source dir to read from and write to
source: PathBuf,
/// How long the responses can/should be cached
time_to_live: Duration,
entries: HashMap<Inode, SampleEntry>,
children: HashMap<Inode, Vec<Inode>>,
/// The generation of the filesystem
/// This is used to invalidate the cache
/// when the filesystem is remounted
generation: u64,
}
impl SampleFilesystem {
pub fn new(root: impl AsRef<Path>, source: impl AsRef<Path>) -> Self {
debug!("new: {:?}; {:?}", root.as_ref(), source.as_ref());
let mut entries = HashMap::new();
// Add root directory with inode number 1
let root_attr = FileAttr {
ino: FUSE_ROOT_ID,
size: 0,
blocks: 0,
atime: UNIX_EPOCH,
mtime: UNIX_EPOCH,
ctime: UNIX_EPOCH,
crtime: UNIX_EPOCH,
kind: FileType::Directory,
perm: 0o755,
nlink: 2,
uid: 0,
gid: 0,
rdev: 0,
blksize: 4096,
flags: 0,
};
entries.insert(
FUSE_ROOT_ID.into(),
SampleEntry::new(
FUSE_ROOT_ID,
"root",
LocalPath::from(root.as_ref()),
root_attr,
),
);
Self {
root: root.as_ref().to_path_buf(),
source: source.as_ref().to_path_buf(),
time_to_live: Duration::from_secs(2),
entries,
/*TODO: implement a way to increase this if necessary*/
generation: 0,
children: HashMap::new(),
}
}
}
#[async_trait::async_trait]
impl CommonFilesystem<SampleEntry> for SampleFilesystem {
fn get_entries(&self) -> &HashMap<Inode, SampleEntry> {
&self.entries
}
fn get_entries_mut(&mut self) -> &mut HashMap<Inode, SampleEntry> {
&mut self.entries
}
fn get_children(&self) -> &HashMap<Inode, Vec<Inode>> {
&self.children
}
fn get_children_mut(&mut self) -> &mut HashMap<Inode, Vec<Inode>> {
&mut self.children
}
fn get_root_path(&self) -> LocalPath {
self.source.clone().into()
}
async fn add_entry(
&mut self,
name: &OsStr,
mode: u16,
file_type: FileType,
parent_ino: impl Into<Inode> + Send,
size: u64,
) -> Result<Inode> {
let parent_ino = parent_ino.into();
let ino = self.generate_ino(); // Generate a new inode number
let now = std::time::SystemTime::now();
let attr = FileAttr {
ino: ino.into(),
size: size,
blocks: 0,
atime: now,
mtime: now,
ctime: now,
crtime: now,
kind: file_type,
perm: mode,
nlink: 1,
uid: 0,
gid: 0,
rdev: 0,
blksize: 4096,
flags: 0,
};
self.get_entries_mut()
.insert(ino, SampleEntry::new(ino, name, OsString::from(name), attr));
self.add_child(parent_ino, &ino);
Ok(ino)
}
}
impl SampleFilesystem {
async fn add_dir_entry(
&mut self,
folder_path: &Path,
parent_ino: impl Into<Inode>,
skip_self: bool,
) -> Result<()> {
let parent_ino = parent_ino.into();
let ino: Inode;
if skip_self {
ino = parent_ino;
} else {
ino = self
.add_entry(
folder_path.file_name().unwrap(),
/*TODO: correct permissions*/
0o755,
FileType::Directory,
parent_ino,
/*TODO: implement size for folders*/ 0,
)
.await?;
}
let d = std::fs::read_dir(folder_path);
if let Ok(d) = d {
for entry in d {
if let Ok(entry) = entry {
let path = entry.path();
let name = entry.file_name();
let metadata = entry.metadata();
if let Ok(metadata) = metadata {
if metadata.is_dir() {
self.add_dir_entry(&path, ino, false);
} else if metadata.is_file() {
let mode = metadata.mode();
let size = metadata.size();
//TODO: async call
// self.add_file_entry(ino, name.as_os_str(), mode as u16, size);
}
}
}
}
}
Ok(())
}
}
impl fuser::Filesystem for SampleFilesystem {
fn init(
&mut self,
_req: &Request<'_>,
_config: &mut KernelConfig,
) -> std::result::Result<(), c_int> {
debug!("init");
// self.add_file_entry(1, "hello.txt".as_ref(), 0o644);
let source = self.source.clone();
run_async_blocking(async {
self.add_dir_entry(&source, FUSE_ROOT_ID, true).await;
});
// self.add_dir_entry(&source, FUSE_ROOT_ID, true);
Ok(())
}
fn lookup(&mut self, _req: &Request<'_>, parent: u64, name: &OsStr, reply: ReplyEntry) {
debug!("lookup: {}:{:?}", parent, name);
for (inode, entry) in self.entries.iter() {
let path: PathBuf = entry.local_path.clone().into();
let accepted = name.eq_ignore_ascii_case(&path);
debug!(
"entry: {}:(accepted={}){:?}; {:?}",
inode, accepted, path, entry.attr
);
if accepted {
reply.entry(&self.time_to_live, &entry.attr, self.generation);
return;
}
}
reply.error(libc::ENOENT);
}
fn getattr(&mut self, _req: &Request<'_>, ino: u64, reply: ReplyAttr) {
self.entries.get(&ino.into()).map(|entry| {
reply.attr(&self.time_to_live, &entry.attr);
});
}
fn readdir(
&mut self,
_req: &Request<'_>,
ino: u64,
fh: u64,
mut offset: i64,
mut reply: ReplyDirectory,
) {
debug!("readdir: {}:{}:{:?}", ino, fh, offset);
let children = self.children.get(&ino.into());
if let Some(attr) = self.get_entries().get(&ino.into()).map(|entry| entry.attr) {
if attr.kind != FileType::Directory {
reply.error(libc::ENOTDIR);
return;
}
}
if !(children.is_none()) {
} else {
reply.error(libc::ENOENT);
return;
}
let children = children.unwrap();
debug!("children ({}): {:?}", children.len(), children);
for child_inode in children.iter().skip(offset as usize) {
let entry = self.entries.get(child_inode).unwrap();
let path: PathBuf = entry.local_path.clone().into();
let attr = entry.attr;
let inode = (*child_inode).into();
offset += 1; // Increment the offset for each processed entry
debug!("entry: {}:{:?}; {:?}", inode, path, attr);
if !reply.add(inode, offset, attr.kind, path) {
break;
}
}
debug!("readdir: ok");
reply.ok();
}
fn read(
&mut self,
_req: &Request<'_>,
ino: u64,
fh: u64,
offset: i64,
size: u32,
flags: i32,
lock_owner: Option<u64>,
reply: ReplyData,
) {
debug!(
"read: {}:{}:{}:{}:{:#x?}:{:?}",
ino, fh, offset, size, flags, lock_owner
);
let data = self.get_entry(ino).map(|entry| entry.attr);
if let Some(attr) = data {
if attr.kind != FileType::RegularFile {
reply.error(libc::EISDIR);
return;
}
let path = self.get_full_path_from_ino(ino);
debug!("opening file: {:?}", &path);
let mut file = std::fs::File::open::<PathBuf>(path.clone().unwrap().into()).unwrap();
let mut buf = vec![0; size as usize];
debug!("reading file: {:?} at {} with size {}", &path, offset, size);
file.read_at(&mut buf, offset as u64).unwrap();
debug!("read file: {:?} at {}", &path, offset);
reply.data(&buf);
} else {
reply.error(libc::ENOENT);
}
}
fn write(
&mut self,
_req: &Request<'_>,
ino: u64,
fh: u64,
offset: i64,
data: &[u8],
write_flags: u32,
flags: i32,
lock_owner: Option<u64>,
reply: ReplyWrite,
) {
debug!(
"write: {}:{}:{}:{:#x?}:{:?}:{:#x?}:{:?}",
ino, fh, offset, flags, lock_owner, write_flags, data,
);
let attr = self.get_entry(ino).map(|entry| entry.attr);
if let Some(attr) = attr {
if attr.kind != FileType::RegularFile {
warn!(
"write: not a file, writing is not supported: kind:{:?}; attr:{:?}",
attr.kind, attr
);
reply.error(libc::EISDIR);
return;
}
let path = self.get_full_path_from_ino(ino);
debug!("opening file: {:?}", &path);
let mut file = OpenOptions::new()
.write(true)
.create(true)
.open::<PathBuf>(path.clone().unwrap().into())
.unwrap();
debug!(
"writing file: {:?} at {} with size {}",
&path,
offset,
data.len()
);
let size = file.write_at(data, offset as u64).unwrap();
debug!("wrote file: {:?} at {}; wrote {} bits", &path, offset, size);
reply.written(size as u32);
} else {
reply.error(libc::ENOENT);
}
}
fn setattr(
&mut self,
_req: &Request<'_>,
ino: u64,
mode: Option<u32>,
uid: Option<u32>,
gid: Option<u32>,
size: Option<u64>,
_atime: Option<TimeOrNow>,
_mtime: Option<TimeOrNow>,
_ctime: Option<SystemTime>,
/*TODO: check if this change need to be implemented*/
fh: Option<u64>,
_crtime: Option<SystemTime>,
/*TODO: check if this change need to be implemented*/
_chgtime: Option<SystemTime>,
/*TODO: check if this change need to be implemented*/
_bkuptime: Option<SystemTime>,
flags: Option<u32>,
reply: ReplyAttr,
) {
debug!(
"setattr: {}:{:?}:{:?}:{:?}:{:?}:{:?}:{:?}:{:?}:{:?}:{:?}:{:?}:{:?}:{:?}",
ino,
mode,
uid,
gid,
size,
_atime,
_mtime,
_ctime,
fh,
_crtime,
_chgtime,
_bkuptime,
flags
);
let attr = self
.entries
.get_mut(&ino.into())
.map(|entry| &mut entry.attr);
if attr.is_none() {
reply.error(libc::ENOENT);
return;
}
let mut attr = attr.unwrap();
if let Some(mode) = mode {
attr.perm = mode as u16;
}
if let Some(uid) = uid {
attr.uid = uid;
}
if let Some(gid) = gid {
attr.gid = gid;
}
if let Some(size) = size {
attr.size = size;
}
if let Some(atime) = _atime {
attr.atime = Self::convert_to_system_time(atime);
}
if let Some(mtime) = _mtime {
attr.mtime = Self::convert_to_system_time(mtime);
}
if let Some(ctime) = _ctime {
attr.ctime = ctime;
}
if let Some(crtime) = _crtime {
attr.crtime = crtime;
}
if let Some(flags) = flags {
attr.flags = flags;
}
reply.attr(&self.time_to_live, attr);
}
fn access(&mut self, _req: &Request<'_>, ino: u64, mask: i32, reply: ReplyEmpty) {
reply.ok(); //TODO: implement this a bit better/more useful
}
}

367
src/google_drive/drive.rs Normal file
View File

@@ -0,0 +1,367 @@
use crate::google_drive::{drive, helpers, DriveId};
use crate::prelude::*;
use std::ffi::{OsStr, OsString};
// use drive3::api::Scope::File;
use anyhow::anyhow;
use drive3::api::{File, Scope};
use drive3::client::ReadSeek;
use drive3::hyper::body::HttpBody;
use drive3::hyper::client::HttpConnector;
use drive3::hyper::{body, Body, Response};
use drive3::hyper_rustls::HttpsConnector;
use drive3::DriveHub;
use drive3::{hyper_rustls, oauth2};
use futures::{Stream, StreamExt};
use hyper::Client;
use log::{debug, trace, warn};
use mime::{FromStrError, Mime};
use std::fmt::{Debug, Error};
use std::io::Write;
use std::path::{Path, PathBuf};
use std::str::FromStr;
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt};
use tokio::runtime::Runtime;
use tokio::{fs, io};
pub struct GoogleDrive {
hub: DriveHub<HttpsConnector<HttpConnector>>,
}
impl GoogleDrive {
pub async fn download_file(&self, file_id: DriveId, target_file: &PathBuf) -> Result<()> {
debug!(
"download_file: file_id: {:50?} to {}",
file_id,
target_file.display()
);
let file_id: String = match file_id.try_into() {
Ok(file_id) => file_id,
Err(e) => return Err(anyhow!("invalid file_id: {:?}", e).into()),
};
let x = download_file_by_id(&self, file_id, target_file.as_path()).await;
debug!("download_file: completed");
let x = x?;
debug!("download_file: success");
Ok(())
}
}
impl GoogleDrive {
pub async fn get_id(&self, path: &OsStr, parent_drive_id: Option<DriveId>) -> Result<DriveId> {
debug!("Get ID of '{:?}' with parent: {:?}", path, parent_drive_id);
let path: OsString = path.into();
let path = match path.into_string() {
Ok(path) => path,
Err(_) => return Err("invalid path".into()),
};
let parent_drive_id: OsString = match parent_drive_id {
Some(parent_drive_id) => parent_drive_id,
None => DriveId::from("root"),
}
.into();
let parent_drive_id = match parent_drive_id.into_string() {
Ok(parent_drive_id) => parent_drive_id,
Err(_) => return Err("invalid parent_drive_id".into()),
};
debug!("get_id: path: {}", path);
debug!("get_id: parent_drive_id: {}", parent_drive_id);
let req = self
.hub
.files()
.list()
.q(&format!(
// "'{}' in parents, '{}' == name",
"name = '{}' and '{}' in parents",
path, parent_drive_id
))
.param("fields", "files(id)")
.doit()
.await;
let (response, files) = match req {
Ok((response, files)) => (response, files),
Err(e) => {
warn!("get_id: Error: {}", e);
return Err("Error".into());
}
};
if files.files.is_none() {
warn!("get_id: No files found (0)");
return Err("No files found".into());
}
let files = files.files.unwrap();
if files.len() == 0 {
warn!("get_id: No files found (1)");
return Err("No files found".into());
}
if files.len() > 1 {
warn!("get_id: Multiple files found");
return Err("Multiple files found".into());
}
let file = files.into_iter().next().unwrap();
let id = file.id.unwrap();
debug!("get_id: id: {}", id);
Ok(DriveId::from(id))
}
}
impl GoogleDrive {
pub(crate) async fn new() -> Result<Self> {
let auth = drive3::oauth2::read_application_secret("auth/client_secret.json").await?;
let auth = oauth2::InstalledFlowAuthenticator::builder(
auth,
oauth2::InstalledFlowReturnMethod::HTTPRedirect,
)
.persist_tokens_to_disk("auth/tokens.json")
.build()
.await?;
let http_client = Client::builder().build(
hyper_rustls::HttpsConnectorBuilder::new()
.with_native_roots()
.https_or_http()
.enable_http1()
.enable_http2()
.build(),
);
let hub = DriveHub::new(http_client, auth);
let mut drive = GoogleDrive { hub };
Ok(drive)
}
pub async fn list_files(&mut self, folder_id: DriveId) -> Result<Vec<File>> {
let folder_id: OsString = folder_id.into();
let folder_id = match folder_id.into_string() {
Ok(folder_id) => folder_id,
Err(_) => return Err("invalid folder_id".into()),
};
if folder_id.is_empty() {
return Err("folder_id is empty".into());
}
if folder_id.contains('\'') {
return Err("folder_id contains invalid character".into());
}
let mut files = Vec::new();
let mut page_token = None;
loop {
let (response, result) = self
.hub
.files()
.list()
.param(
"fields",
"nextPageToken, files(id, name, size, mimeType, kind)",
)
// .page_token(page_token.as_ref().map(String::as_str))
.q(format!("'{}' in parents", folder_id).as_str())
.doit()
.await?;
files.extend(result.files.ok_or("no file list returned")?);
page_token = result.next_page_token;
if page_token.is_none() {
break;
}
}
Ok(files)
}
}
impl Debug for GoogleDrive {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "GoogleDrive")
}
}
pub async fn sample() -> Result<()> {
debug!("sample");
let mut drive = GoogleDrive::new().await?;
sample_list_files(&mut drive).await?;
let hello_world_file = get_files_by_name(&mut drive, "hello_world.txt").await?;
let hello_world_file = hello_world_file
.first()
.ok_or("hello_world.txt not found")?;
debug!("hello_world_file: id:{:?}", hello_world_file.id);
let target_path = "/tmp/hello_world.txt";
let target_path = std::path::Path::new(target_path);
// download_file(&mut drive, hello_world_file, target_path).await?;
debug!("target_path: {:?}", target_path);
debug!("download_file_by_id");
let hello_world_file_id = hello_world_file.id.as_ref().ok_or("")?;
download_file_by_id(&mut drive, hello_world_file_id, target_path).await?;
debug!("get_file_header_by_id");
get_file_header_by_id(&mut drive, hello_world_file_id).await?;
debug!("done");
Ok(())
}
async fn download_file(
hub: &GoogleDrive,
file: &drive3::api::File,
target_path: &Path,
) -> Result<File> {
if let Some(id) = &file.id {
download_file_by_id(hub, id, target_path).await
} else {
Err("file id not found".into())
}
}
async fn download_file_by_id(
hub: &GoogleDrive,
id: impl Into<String>,
target_path: &Path,
) -> Result<File> {
use tokio::fs::File;
use tokio::io::AsyncWriteExt;
let (response, content): (Response<Body>, google_drive3::api::File) = hub
.hub
.files()
.get(&id.into())
.add_scope(Scope::Readonly)
.acknowledge_abuse(true)
.param("alt", "media")
.doit()
.await?;
//TODO: bigger files don't get downloaded. it just starts and then hangs at ~1.3MB forever
debug!("download_file_by_id(): response: {:?}", response);
debug!("download_file_by_id(): content: {:?}", content);
write_body_to_file(response, target_path).await?;
Ok(content)
}
async fn write_body_to_file(response: Response<Body>, target_path: &Path) -> Result<()> {
use futures::StreamExt;
debug!("write_body_to_file(): target_path: {:?}", target_path);
let mut file = std::fs::File::create(target_path)?;
let mut stream = response.into_body();
let mut buffer = bytes::BytesMut::new();
let mut counter = 0;
while let Some(chunk) = stream.next().await {
let chunk = chunk?;
trace!("write_body_to_file(): chunk counter: {}", counter);
file.write_all(&chunk)?;
counter += 1;
}
debug!("write_body_to_file(): done");
Ok(())
}
async fn get_file_header_by_id(hub: &GoogleDrive, id: &str) -> Result<File> {
debug!("get_file_header_by_id(): id: {:?}", id);
let (response, content) = hub.hub.files().get(id).doit().await?;
Ok(content)
}
async fn get_files_by_name(
drive: &GoogleDrive,
name: impl Into<String>,
) -> Result<Vec<drive3::api::File>> {
let name = name.into();
if name.is_empty() {
return Err("name cannot be empty".into());
}
if name.contains("'") {
return Err("name cannot contain single quote".into());
}
let (response, files) = drive
.hub
.files()
.list()
.q(format!("name = '{}'", name).as_str())
.doit()
.await?;
debug!("get_files_by_name(): response: {:?}", response);
debug!("get_files_by_name(): files: {:?}", files);
let files: Vec<drive3::api::File> = files.files.unwrap_or(vec![]);
Ok(files)
}
async fn sample_list_files(drive: &GoogleDrive) -> Result<()> {
let (hello_world_res, hello_world_list) = drive
.hub
.files()
.list()
// .q("name = 'hello_world.txt'")
// .q("'root' in parents and trashed=false")
.doit()
.await?;
debug!("hello_world_res: {:?}", hello_world_res);
debug!("hello_world_list: {:?}", hello_world_list);
let files: Vec<drive3::api::File> = hello_world_list.files.unwrap_or(vec![]);
debug!("hello_world_list amount of files: {}", files.len());
for file in files {
let name = file.name.unwrap_or("NO NAME".to_string());
let id = file.id.unwrap_or("NO ID".to_string());
let kind = file.kind.unwrap_or("NO KIND".to_string());
let mime_type = file.mime_type.unwrap_or("NO MIME TYPE".to_string());
debug!(
"file: {:100}name:{:100}kind: {:25}mime_type: {:100}",
id, name, kind, mime_type
);
}
Ok(())
}
async fn create_file_on_drive_from_path(
drive: &GoogleDrive,
file: File,
path: &Path,
mime_type: mime::Mime,
) -> Result<()> {
let content = fs::File::open(path).await?;
create_file_on_drive(drive, file, mime_type, content).await?;
Ok(())
}
async fn create_file_on_drive(
drive: &GoogleDrive,
file: google_drive3::api::File,
mime_type: mime::Mime,
content: tokio::fs::File,
) -> Result<drive3::api::File> {
let stream = content.into_std().await;
let (response, file) = drive
.hub
.files()
.create(file)
.upload_resumable(stream, mime_type)
.await?;
debug!("create_file(): response: {:?}", response);
debug!("create_file(): file: {:?}", file);
Ok(file)
}
pub async fn update_file_content_on_drive_from_path(
drive: &GoogleDrive,
file: google_drive3::api::File,
source_path: &Path,
) -> Result<()> {
Ok(())
}
async fn update_file_content_on_drive(
drive: &GoogleDrive,
file: google_drive3::api::File,
content: fs::File,
) -> Result<()> {
let stream = content.into_std().await;
let mime_type = helpers::get_mime_from_file_metadata(&file)?;
let id = file.id.clone().unwrap();
let (response, file) = drive
.hub
.files()
.update(file, &id)
.upload(stream, mime_type)
.await?;
debug!("update_file_on_drive(): response: {:?}", response);
debug!("update_file_on_drive(): file: {:?}", file);
Ok(())
}

43
src/google_drive/drive_id.rs Normal file
View File

@@ -0,0 +1,43 @@
use std::ffi::OsString;
use std::fmt::Display;
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct DriveId(OsString);
impl DriveId {
pub(crate) fn root() -> DriveId {
DriveId(OsString::from("root"))
}
}
impl Into<OsString> for DriveId {
fn into(self) -> OsString {
self.0
}
}
impl TryInto<String> for DriveId {
type Error = OsString;
fn try_into(self) -> Result<String, Self::Error> {
self.0.into_string()
}
}
impl From<OsString> for DriveId {
fn from(value: OsString) -> Self {
DriveId(value)
}
}
impl From<String> for DriveId {
fn from(value: String) -> Self {
OsString::from(value).into()
}
}
impl From<&str> for DriveId {
fn from(s: &str) -> Self {
DriveId(OsString::from(s))
}
}
impl DriveId {
pub fn new(id: impl Into<OsString>) -> Self {
Self(id.into())
}
}

69
src/google_drive/helpers.rs Normal file
View File

@@ -0,0 +1,69 @@
use crate::fs::drive::DriveFilesystem;
use crate::fs::{CommonFilesystem, Inode};
use crate::google_drive::{DriveId, GoogleDrive};
use crate::prelude::*;
use anyhow::anyhow;
use drive3::api::File;
use log::debug;
use mime::Mime;
use std::path::{Path, PathBuf};
use std::str::FromStr;
pub fn get_mime_from_file_metadata(file: &File) -> Result<Mime> {
Ok(Mime::from_str(
&file.mime_type.as_ref().unwrap_or(&"*/*".to_string()),
)?)
}
pub fn get_drive_id_from_local_path(drive: &DriveFilesystem, path: &Path) -> Result<DriveId> {
let drive_mount_point: &PathBuf = &drive.get_root_path().into();
debug!("get_drive_id_from_path(): (0) path: '{}'", path.display());
let path = match path.strip_prefix(drive_mount_point) {
Err(e) => {
return Err(anyhow!(
"Path {:?} is not a prefix of {:?}",
drive_mount_point,
path
))?
}
Ok(path) => path,
};
debug!("get_drive_id_from_path(): (1) path: '{}'", path.display());
if path == Path::new("/") || path == Path::new("") {
debug!(
"get_drive_id_from_path(): (1) path is root: '{}'",
path.display()
);
return Ok("root".into());
}
let mut parent_ino: Inode = 5u32.into();
// let mut parent_ino : Inode =Inode::from(5u32);//.into();
for part in path.iter() {
debug!("get_drive_id_from_path(): (2..) path: '{:?}'", part);
let children = drive.get_children().get(&parent_ino);
debug!("get_drive_id_from_path(): (2..) children: '{:?}'", children);
}
todo!("get_drive_id_from_path()")
}
mod test {
use super::*;
#[tokio::test]
async fn test_get_drive_id_from_local_path() {
crate::init_logger();
let path = Path::new("/drive1");
let drive = DriveFilesystem::new(path).await;
let drive_mount_point = Path::new("/drive1");
let drive_id = get_drive_id_from_local_path(&drive, path).unwrap();
assert_eq!(drive_id, "root".into());
let path = Path::new("/drive1/");
let drive_id = get_drive_id_from_local_path(&drive, path).unwrap();
assert_eq!(drive_id, "root".into());
let path = Path::new("/drive1/dir1/dir2/file1.txt");
let drive_id = get_drive_id_from_local_path(&drive, path).unwrap();
todo!("create assert for this test");
// assert_eq!(drive_id, "TODO".into());
}
}

8
src/google_drive/mod.rs Normal file
View File

@@ -0,0 +1,8 @@
mod helpers;
pub use drive::*;
pub use helpers::*;
mod drive;
mod drive_id;
pub use drive_id::*;

476
src/lib.rs Normal file
View File

@@ -0,0 +1,476 @@
#![allow(dead_code, unused)]
use google_drive3::oauth2::read_application_secret;
use std::error::Error;
use std::time::{Duration, SystemTime};
extern crate google_drive3 as drive3;
use drive3::api::Channel;
use drive3::{hyper, hyper_rustls, oauth2, DriveHub};
use log::{debug, error, info, trace, warn};
// use nix;
use notify::{recommended_watcher, INotifyWatcher, RecommendedWatcher};
use tokio::io::{stdin, AsyncReadExt};
use tokio::sync::mpsc::channel;
pub mod async_helper;
pub mod common;
pub mod fs;
pub mod google_drive;
pub mod prelude;
use prelude::*;
#[cfg(test)]
mod tests {
use super::*;
fn init_logger() {
let _ = env_logger::builder()
.filter_level(log::LevelFilter::Trace)
.is_test(true)
.try_init();
}
#[tokio::test]
async fn does_it_work() {
init_logger();
list_files().await;
}
}
pub fn init_logger() {
env_logger::builder()
.filter_level(log::LevelFilter::Debug)
.try_init();
}
pub async fn sample() -> Result<()> {
//Test file id: "1IotISYu3cF7JrOdfFPKNOkgYg1-ii5Qs"
list_files().await
}
async fn list_files() -> Result<()> {
debug!("Hello, world!");
let secret: oauth2::ApplicationSecret = read_application_secret("auth/client_secret.json")
.await
.expect("failed to read client secret file");
let auth = oauth2::InstalledFlowAuthenticator::builder(
secret,
oauth2::InstalledFlowReturnMethod::HTTPRedirect,
)
.persist_tokens_to_disk("auth/token_store.json")
.build()
.await?;
let hub = DriveHub::new(
hyper::Client::builder().build(
hyper_rustls::HttpsConnectorBuilder::new()
.with_native_roots()
.https_or_http()
.enable_http1()
.enable_http2()
.build(),
),
auth,
);
let result = hub
.files()
.get("1IotISYu3cF7JrOdfFPKNOkgYg1-ii5Qs")
.doit()
.await?;
// debug!("Result: {:?}", result);
let (body, file) = result;
debug!("Body: {:?}", body);
debug!("File: {:?}", file);
// let result = hub.files().list().corpus("user").doit().await;
// debug!("Result: {:?}", result);
info!("Filename: {:?}", file.name.unwrap_or("NO NAME".to_string()));
info!(
"Description: {:?}",
file.description.unwrap_or("NO DESCRIPTION".to_string())
);
Ok(())
}
use fuser::{
FileAttr, FileType, Filesystem, MountOption, ReplyAttr, ReplyData, ReplyDirectory, ReplyEmpty,
ReplyEntry, ReplyOpen, ReplyWrite, ReplyXattr, Request, TimeOrNow, FUSE_ROOT_ID,
};
use std::ffi::OsStr;
use std::path::Path;
use std::time::UNIX_EPOCH;
#[derive(Default)]
struct MyFS {
/// how long the responses can/should be cached
time_to_live: Duration,
main_ino: u64,
main_size: u64,
main_blksize: u64,
main_uid: u32,
main_gid: u32,
main_flags: u32,
main_content: Vec<u8>,
main_file_type: Option<FileType>,
main_name: String,
}
use crate::fs::drive::DriveFilesystem;
use crate::fs::sample::SampleFilesystem;
use async_trait::async_trait;
use tokio::runtime::Runtime;
struct DirEntry {
ino: u64,
name: String,
file_type: FileType,
}
impl MyFS {
fn get_attr(&self, ino: u64) -> Option<FileAttr> {
// Get the file attributes based on the inode number
if ino == FUSE_ROOT_ID {
Some(FileAttr {
ino: FUSE_ROOT_ID,
size: 0,
blocks: 0,
atime: UNIX_EPOCH,
mtime: UNIX_EPOCH,
ctime: UNIX_EPOCH,
crtime: UNIX_EPOCH,
kind: FileType::Directory,
perm: 0o755,
nlink: 0,
uid: 0,
gid: 0,
rdev: 0,
blksize: 0,
flags: 0,
})
} else if ino == self.main_ino {
Some(FileAttr {
ino: FUSE_ROOT_ID,
size: self.main_size,
blocks: 0,
atime: UNIX_EPOCH,
mtime: UNIX_EPOCH,
ctime: UNIX_EPOCH,
crtime: UNIX_EPOCH,
kind: self.main_file_type.unwrap_or(FileType::RegularFile),
perm: 0o755,
nlink: 0,
uid: self.main_uid,
gid: self.main_gid,
rdev: 0,
blksize: self.main_blksize as u32,
flags: self.main_flags,
})
} else {
None
}
}
fn set_attr(
&mut self,
ino: u64,
mode: Option<u32>,
uid: Option<u32>,
gid: Option<u32>,
size: Option<u64>,
flags: Option<u32>,
) -> Option<FileAttr> {
debug!(
"set_attr=> ino: {}; mode: {:?}; uid: {:?}; gid: {:?}; size: {:?}; flags: {:?}",
ino, mode, uid, gid, size, flags
);
// Get the file attributes based on the inode number
if ino == self.main_ino {
self.main_size = size.unwrap_or(self.main_size);
self.main_flags = flags.unwrap_or(self.main_flags);
self.main_uid = uid.unwrap_or(self.main_uid);
self.main_gid = gid.unwrap_or(self.main_gid);
return self.get_attr(ino);
} else {
None
}
}
fn write_file(
&mut self,
ino: u64,
fh: u64,
offset: i64,
data: &[u8],
flags: i32,
) -> Option<usize> {
// Write the file and reply with the number of bytes written
debug!(
"write_file=> ino: {}; fh: {}; offset: {}; data: {:?}; flags: {}",
ino, fh, offset, data, flags
);
if ino == self.main_ino {
self.main_content = data.to_vec();
// todo!("write the file and reply with the number of bytes written");
return Some(data.len());
} else {
None
}
}
fn read_file(&self, ino: u64, fh: u64, offset: i64, size: u32) -> Option<Vec<u8>> {
debug!(
"read_file=> ino: {}; fh: {}; offset: {}; size: {}",
ino, fh, offset, size
);
if ino == self.main_ino {
// Read the file and reply with the data
let data = &self.main_content.clone(); //b"Hello World!";
let offset_usize = offset as usize;
let size_usize = size as usize;
if data.len() <= offset_usize {
let result = vec![libc::EOF as u8];
debug!("read_file=> (0) result: {:?}", result);
return Some(result);
}
if offset_usize + size_usize > data.len() {
//return the rest of the data + EOF
let mut result = data[1..].to_vec();
result.push(libc::EOF as u8);
debug!("read_file=> (1) result: {:?}", result);
return Some(result);
// todo!("output the rest of the data + EOF, not just EOF");
return None;
}
let result = data[offset_usize..offset_usize + size_usize].to_vec();
debug!("read_file=> (2) result: {:?}", result);
return Some(result);
} else {
None
}
}
fn read_dir(&self, ino: u64) -> Option<Vec<DirEntry>> {
if ino == FUSE_ROOT_ID {
let mut entries = Vec::new();
let dir_entry = DirEntry {
ino: self.main_ino,
name: self.main_name.clone(),
file_type: self.main_file_type.unwrap_or(FileType::RegularFile),
};
entries.push(dir_entry);
Some(entries)
} else {
None
}
}
}
#[async_trait]
impl Filesystem for MyFS {
fn open(&mut self, _req: &Request<'_>, _ino: u64, _flags: i32, reply: ReplyOpen) {
if _ino == self.main_ino {
reply.opened(0, 0);
} else {
reply.error(libc::ENOENT);
}
}
fn access(&mut self, _req: &Request<'_>, ino: u64, mask: i32, reply: ReplyEmpty) {
if ino == self.main_ino {
reply.ok()
} else {
reply.error(libc::ENOENT)
}
}
fn getattr(&mut self, _req: &Request, ino: u64, reply: ReplyAttr) {
if let Some(attr) = self.get_attr(ino) {
reply.attr(&self.time_to_live, &attr);
} else {
reply.error(libc::ENOENT);
}
}
fn write(
&mut self,
_req: &Request<'_>,
ino: u64,
fh: u64,
offset: i64,
data: &[u8],
write_flags: u32,
flags: i32,
lock_owner: Option<u64>,
reply: ReplyWrite,
) {
if let Some(size) = self.write_file(ino, fh, offset, data, flags) {
reply.written(size as u32);
} else {
reply.error(libc::ENOENT);
}
}
fn setattr(
&mut self,
_req: &Request<'_>,
ino: u64,
mode: Option<u32>,
uid: Option<u32>,
gid: Option<u32>,
size: Option<u64>,
_atime: Option<TimeOrNow>,
_mtime: Option<TimeOrNow>,
_ctime: Option<SystemTime>,
fh: Option<u64>,
_crtime: Option<SystemTime>,
_chgtime: Option<SystemTime>,
_bkuptime: Option<SystemTime>,
flags: Option<u32>,
reply: ReplyAttr,
) {
if let Some(attr) = self.set_attr(ino, mode, uid, gid, size, flags) {
reply.attr(&self.time_to_live, &attr);
} else {
reply.error(libc::ENOENT);
}
}
fn read(
&mut self,
_req: &Request<'_>,
ino: u64,
fh: u64,
offset: i64,
size: u32,
flags: i32,
lock_owner: Option<u64>,
reply: ReplyData,
) {
if let Some(data) = self.read_file(ino, fh, offset, size) {
let data = data.as_slice();
reply.data(data);
} else {
reply.error(libc::ENOENT);
}
}
fn readdir(
&mut self,
_req: &Request<'_>,
ino: u64,
fh: u64,
offset: i64,
mut reply: ReplyDirectory,
) {
debug!("readdir=> ino: {}; fh: {}; offset: {}", ino, fh, offset);
if let Some(entries) = self.read_dir(ino) {
for (i, entry) in entries.iter().enumerate().skip(offset as usize) {
if reply.add(entry.ino, (i + 1) as i64, entry.file_type, &entry.name) {
break;
}
}
reply.ok();
} else {
reply.error(libc::ENOENT);
}
}
fn lookup(&mut self, _req: &Request<'_>, parent: u64, name: &OsStr, reply: ReplyEntry) {
let main_path = OsStr::new(&self.main_name);
debug!(
"lookup=> parent: {}; name: {:?}; main_path: {:?}",
parent, name, main_path
);
if name.eq_ignore_ascii_case(main_path) {
let attr = self.get_attr(self.main_ino).unwrap();
reply.entry(&self.time_to_live, &attr, 0);
} else {
reply.error(libc::ENOENT);
}
}
}
pub async fn watch_file_reading() -> Result<()> {
let mountpoint = "/tmp/fuse/1";
let options = vec![
MountOption::RW,
// MountOption::FSName("myfs".to_string()),
// MountOption::AllowOther,
// MountOption::AutoUnmount,
];
debug!("Mounting fuse filesystem at {}", mountpoint);
fuser::mount2(
MyFS {
time_to_live: Duration::from_secs(5),
main_ino: 2,
main_name: "1.txt".to_string(),
main_file_type: Some(FileType::RegularFile),
main_content: b"Hello World!".to_vec(),
..Default::default()
},
mountpoint,
&options,
)
.unwrap();
debug!("Exiting...");
Ok(())
}
pub async fn sample_fs() -> Result<()> {
let mountpoint = "/tmp/fuse/1";
let source = "/tmp/fuse/2";
let options = vec![MountOption::RW];
debug!("Mounting fuse filesystem at {}", mountpoint);
let fs = SampleFilesystem::new(mountpoint, source);
fuser::mount2(fs, mountpoint, &options).unwrap();
debug!("Exiting...");
Ok(())
}
pub async fn sample_drive_fs() -> Result<()> {
let mountpoint = "/tmp/fuse/3";
// let source = "/tmp/fuse/2";
let options = vec![MountOption::RW];
debug!("Mounting fuse filesystem at {}", mountpoint);
let fs = DriveFilesystem::new(mountpoint).await?;
fuser::mount2(fs, mountpoint, &options).unwrap();
debug!("Exiting...");
Ok(())
}
// pub async fn watch_file_reading() -> Result<()> {
// let temp_file = tempfile::NamedTempFile::new()?;
// let file_path = temp_file.path();
// info!("File path: {:?}", file_path);
// use notify::{recommended_watcher, RecursiveMode, Watcher};
// let mut config = notify::Config::default();
// let mut watcher: INotifyWatcher = Watcher::new(MyReadHandler, config).unwrap();
// watcher
// .watch(file_path, RecursiveMode::NonRecursive)
// .unwrap();
//
// info!("Press any key to exit...");
// let x = &mut [0u8; 1];
// stdin().read(x).await?;
// debug!("Done");
// Ok(())
// }
// struct MyReadHandler;
// impl notify::EventHandler for MyReadHandler {
// fn handle_event(&mut self, event: std::result::Result<notify::Event, notify::Error>) {
// debug!("File read: {:?}", event);
// }
// }
//
// pub async fn sample_nix() -> Result<()> {
// info!("Hello, world! (nix)");
// let tmppath = tempfile::tempdir()?;
// nix::mount::mount(
// // Some("/home/omgeeky/Documents/testmount/"),
// None::<&str>,
// tmppath.path(),
// Some("tmpfs"),
// nix::mount::MsFlags::empty(),
// None::<&str>,
// );
// info!("Mounted tmpfs at {:?}", tmppath.path());
// info!("Press any key to exit (nix)...");
// // block execution until keyboard input is received
// nix::unistd::read(0, &mut [0])?;
// nix::mount::umount(tmppath.path()).unwrap();
// info!("Done (nix)");
// Ok(())
// }

21
src/main.rs Normal file
View File

@@ -0,0 +1,21 @@
#[tokio::main]
async fn main() {
untitled::init_logger();
// use tokio::runtime::Runtime;
// let rt = Runtime::new().unwrap();
// let filesystem_runtime = Runtime::new().unwrap();
//
// let handle = rt.handle();
// handle.block_on(async {
// untitled::sample().await.unwrap();
// untitled::sample_fs().await.unwrap();
// untitled::google_drive::sample().await.unwrap();
// untitled::watch_file_reading().await.unwrap();
// untitled::sample_nix().await.unwrap();
untitled::sample_drive_fs().await.unwrap();
// });
// RUNTIME.block_on(async {
// //test
// });
}

3
src/prelude.rs Normal file
View File

@@ -0,0 +1,3 @@
use std::error::Error;
pub type Result<T> = core::result::Result<T, Box<dyn Error>>;