From faf49050e7950348779207af9e681530d26113af Mon Sep 17 00:00:00 2001 From: OMGeeky Date: Sun, 14 May 2023 17:20:38 +0200 Subject: [PATCH] streaming works, is slow but works. no writing yet --- .gitignore | 4 + Cargo.toml | 26 ++ src/async_helper.rs | 18 ++ src/common.rs | 81 ++++++ src/fs/common.rs | 159 ++++++++++ src/fs/drive/entry.rs | 53 ++++ src/fs/drive/mod.rs | 546 +++++++++++++++++++++++++++++++++++ src/fs/inode.rs | 56 ++++ src/fs/mod.rs | 6 + src/fs/sample.rs | 474 ++++++++++++++++++++++++++++++ src/google_drive/drive.rs | 367 +++++++++++++++++++++++ src/google_drive/drive_id.rs | 43 +++ src/google_drive/helpers.rs | 69 +++++ src/google_drive/mod.rs | 8 + src/lib.rs | 476 ++++++++++++++++++++++++++++++ src/main.rs | 21 ++ src/prelude.rs | 3 + 17 files changed, 2410 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.toml create mode 100644 src/async_helper.rs create mode 100644 src/common.rs create mode 100644 src/fs/common.rs create mode 100644 src/fs/drive/entry.rs create mode 100644 src/fs/drive/mod.rs create mode 100644 src/fs/inode.rs create mode 100644 src/fs/mod.rs create mode 100644 src/fs/sample.rs create mode 100644 src/google_drive/drive.rs create mode 100644 src/google_drive/drive_id.rs create mode 100644 src/google_drive/helpers.rs create mode 100644 src/google_drive/mod.rs create mode 100644 src/lib.rs create mode 100644 src/main.rs create mode 100644 src/prelude.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..48c6b02 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/target +/Cargo.lock +/auth +/.idea diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..cb75551 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "untitled" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +google-drive3 = "5.0" +tokio = { version = "1.28", features = ["full"] } +log = "0.4" +env_logger = "0.10" +#nix = { version = "0.26", features = ["mount"] } +tempfile = "3.5.0" +notify = { version = "5.1", default-features = false, features = ["macos_kqueue"]} + +fuser = "0.12" +async-trait = "0.1.68" +libc = "0.2" +reqwest = "0.11.17" +bytes = "1.4.0" +futures = "0.3.28" +hyper = { version = "0.14.24", features = ["full", "stream"] } +mime = "0.3" +anyhow = "1.0" +async-recursion = "1.0.4" \ No newline at end of file diff --git a/src/async_helper.rs b/src/async_helper.rs new file mode 100644 index 0000000..62ff21f --- /dev/null +++ b/src/async_helper.rs @@ -0,0 +1,18 @@ +use log::{debug, trace}; +use std::future::Future; +use tokio::runtime::{Handle, Runtime}; + +/// Run a future to completion on the current thread. +/// This is useful when you want to run a future in a blocking context. +/// This function will block the current thread until the provided future has run to completion. +/// +/// # Be careful with deadlocks +pub fn run_async_blocking(f: impl std::future::Future + Sized) -> T { + trace!("run_async"); + let handle = Handle::current(); + handle.enter(); + trace!("run_async: entered handle"); + let result = futures::executor::block_on(f); + trace!("run_async: got result"); + result +} diff --git a/src/common.rs b/src/common.rs new file mode 100644 index 0000000..2e29b57 --- /dev/null +++ b/src/common.rs @@ -0,0 +1,81 @@ +use std::ffi::{OsStr, OsString}; +use std::ops::Deref; +use std::path::{Path, PathBuf}; + +#[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq, Hash)] +pub struct LocalPath(PathBuf); + +impl From for LocalPath { + fn from(path: PathBuf) -> Self { + Self(path) + } +} + +impl From<&Path> for LocalPath { + fn from(path: &Path) -> Self { + Self(path.to_path_buf()) + } +} + +impl From<&PathBuf> for LocalPath { + fn from(path: &PathBuf) -> Self { + Self(path.to_path_buf()) + } +} + +impl From for LocalPath { + fn from(path: OsString) -> Self { + Self::from(&path) + } +} +impl From<&OsString> for LocalPath { + fn from(path: &OsString) -> Self { + Path::new(path).into() + } +} + +impl AsRef for LocalPath +where + T: ?Sized, + ::Target: AsRef, +{ + fn as_ref(&self) -> &T { + self.0.deref().as_ref() + } +} +impl Deref for LocalPath { + type Target = PathBuf; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} +//------------------------------------------ + +impl Into for LocalPath { + fn into(self) -> PathBuf { + self.0 + } +} +impl Into for LocalPath { + fn into(self) -> OsString { + self.0.into_os_string() + } +} +impl<'a> Into<&'a Path> for &'a LocalPath { + fn into(self) -> &'a Path { + &self.0 + } +} + +impl<'a> Into<&'a OsStr> for &'a LocalPath { + fn into(self) -> &'a OsStr { + self.0.as_os_str() + } +} + +impl<'a> Into<&'a PathBuf> for &'a LocalPath { + fn into(self) -> &'a PathBuf { + &self.0 + } +} diff --git a/src/fs/common.rs b/src/fs/common.rs new file mode 100644 index 0000000..f90e4aa --- /dev/null +++ b/src/fs/common.rs @@ -0,0 +1,159 @@ +use crate::async_helper::run_async_blocking; +use crate::common::LocalPath; +use crate::fs::inode::Inode; +use crate::google_drive::DriveId; +use crate::prelude::*; +use anyhow::anyhow; +use async_trait::async_trait; +use fuser::{FileAttr, FileType, TimeOrNow, FUSE_ROOT_ID}; +use log::debug; +use std::collections::HashMap; +use std::ffi::{OsStr, OsString}; +use std::path::{Path, PathBuf}; +use std::time::SystemTime; + +pub trait CommonEntry { + fn get_ino(&self) -> Inode; + fn get_name(&self) -> &OsStr; + fn get_local_path(&self) -> &LocalPath; + fn get_attr(&self) -> &FileAttr; + + // fn new( + // ino: impl Into, + // name: impl Into, + // local_path: impl Into, + // attr: FileAttr, + // ) -> Self; +} +#[async_trait] +pub trait CommonFilesystem { + fn get_entries(&self) -> &HashMap; + fn get_entries_mut(&mut self) -> &mut HashMap; + fn get_children(&self) -> &HashMap>; + fn get_children_mut(&mut self) -> &mut HashMap>; + fn get_root_path(&self) -> LocalPath; + + fn generate_ino(&self) -> Inode { + Inode::new(self.get_entries().len() as u64 + 1) //TODO: check if this is working or if concurrency is a problem + } + + fn get_path_from_ino(&self, ino: impl Into) -> Option { + let ino = ino.into(); + debug!("get_path_from_ino: {}", ino); + let res = self.get_entry(ino)?.get_local_path().clone(); + debug!("get_path_from_ino: {}:{:?}", ino, res); + Some(res) + } + + fn get_full_path_from_ino(&self, ino: impl Into) -> Option { + let ino = ino.into(); + debug!("get_full_path_from_ino: {}", ino); + if ino == FUSE_ROOT_ID.into() { + return Some(self.get_root_path()); + } + let parent = self.get_parent_ino(ino); + if let Some(parent) = parent { + let path: PathBuf = self.get_full_path_from_ino(parent)?.into(); + let buf: LocalPath = path + .join::(self.get_path_from_ino(ino)?.into()) + .into(); + debug!("get_full_path_from_ino: {}:{:?}", ino, buf); + return Some(buf); + } + match self.get_path_from_ino(ino) { + Some(path) => Some(path.clone()), + None => None, + } + } + + fn get_child_with_path( + &self, + parent: impl Into, + path: impl AsRef, + ) -> Option { + let parent = parent.into(); + let path = path.as_ref(); + debug!("get_child_with_path: {}:{:?}", parent, path); + let children = self.get_children().get(&parent)?; + let mut res = None; + for child in children { + let child_path: &OsStr = self.get_entry(*child)?.get_local_path().into(); + if child_path == path { + res = Some(*child); + break; + } + } + debug!("get_child_with_path: {}:{:?}", parent, res); + res + } + + fn get_parent_ino(&self, ino: impl Into) -> Option { + let ino = ino.into(); + debug!("get_parent_ino: {}", ino); + if ino == FUSE_ROOT_ID.into() { + return None; + } + let mut parent = None; + for (parent_ino, child_inos) in self.get_children().iter() { + if child_inos.contains(&ino) { + parent = Some(*parent_ino); + break; + } + } + parent + } + + fn convert_to_system_time(mtime: TimeOrNow) -> SystemTime { + let mtime = match mtime { + TimeOrNow::SpecificTime(t) => t, + TimeOrNow::Now => SystemTime::now(), + }; + mtime + } + + fn get_entry(&self, ino: impl Into) -> Option<&Entry> { + self.get_entries().get(&ino.into()) + } + fn get_entry_r(&self, ino: impl Into) -> Result<&Entry> { + self.get_entries() + .get(&ino.into()) + .ok_or(anyhow!("Entry not found").into()) + } + + async fn add_file_entry( + &mut self, + parent: impl Into + Send, + name: &OsStr, + mode: u16, + size: u64, + ) -> Result { + let parent = parent.into(); + debug!("add_file_entry: {}:{:?}; {}", parent, name, mode); + + let ino = self + .add_entry(name, mode, FileType::RegularFile, parent, size) + .await?; + + Ok(ino) + } + + async fn add_entry( + &mut self, + name: &OsStr, + mode: u16, + file_type: FileType, + parent_ino: impl Into + Send, + size: u64, + ) -> Result; + + fn add_child(&mut self, parent_ino: impl Into, ino: impl Into) { + let parents_child_list = self + .get_children_mut() + .entry(parent_ino.into()) + .or_default(); + let ino: Inode = ino.into(); + if !parents_child_list.contains(&ino) { + parents_child_list.push(ino); + } + } +} diff --git a/src/fs/drive/entry.rs b/src/fs/drive/entry.rs new file mode 100644 index 0000000..e4b14f7 --- /dev/null +++ b/src/fs/drive/entry.rs @@ -0,0 +1,53 @@ +use crate::common::LocalPath; +use crate::fs::{CommonEntry, Inode}; +use crate::google_drive::DriveId; +use fuser::FileAttr; +use std::ffi::{OsStr, OsString}; +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DriveEntry { + pub ino: Inode, + pub drive_id: DriveId, + + pub name: OsString, + // pub drive_path: OsString, + pub local_path: LocalPath, + pub attr: FileAttr, +} +impl DriveEntry { + pub fn new( + ino: impl Into, + name: impl Into, + drive_id: impl Into, + + local_path: impl Into, + attr: FileAttr, + ) -> Self { + let name = name.into(); + let path = local_path.into(); + Self { + ino: ino.into(), + drive_id: drive_id.into(), + name, + // drive_path: path.clone().into(), + local_path: path, + attr, + } + } +} +impl CommonEntry for DriveEntry { + fn get_ino(&self) -> Inode { + self.ino + } + + fn get_name(&self) -> &OsStr { + &self.name + } + + fn get_local_path(&self) -> &LocalPath { + &self.local_path + } + + fn get_attr(&self) -> &FileAttr { + &self.attr + } +} diff --git a/src/fs/drive/mod.rs b/src/fs/drive/mod.rs new file mode 100644 index 0000000..d662144 --- /dev/null +++ b/src/fs/drive/mod.rs @@ -0,0 +1,546 @@ +use crate::common::LocalPath; +use crate::fs::common::CommonFilesystem; +use crate::fs::inode::Inode; +use crate::fs::CommonEntry; +use crate::google_drive::{DriveId, GoogleDrive}; +use crate::prelude::*; +use crate::prelude::*; +use anyhow::{anyhow, Error}; +use async_recursion::async_recursion; +use drive3::api::File; +use fuser::{ + FileAttr, FileType, Filesystem, KernelConfig, ReplyAttr, ReplyData, ReplyDirectory, ReplyEmpty, + ReplyEntry, ReplyOpen, ReplyStatfs, ReplyWrite, ReplyXattr, Request, TimeOrNow, FUSE_ROOT_ID, +}; +use futures::TryFutureExt; +use libc::c_int; +use log::{debug, error, warn}; +use mime::Mime; +use std::any::Any; +use std::collections::HashMap; +use std::ffi::{OsStr, OsString}; +use std::fmt::Display; +use std::fs::OpenOptions; +use std::os::unix::prelude::*; +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use tempfile::TempDir; +use tokio::io::{stdin, AsyncBufReadExt}; +use tokio::runtime::Runtime; +mod entry; +use crate::async_helper::run_async_blocking; +pub use entry::*; + +#[derive(Debug)] +pub struct DriveFilesystem { + // runtime: Runtime, + /// the point where the filesystem is mounted + root: PathBuf, + /// the source dir to read from and write to + source: GoogleDrive, + /// the cache dir to store the files in + cache_dir: Option, + + /// How long the responses can/should be cached + time_to_live: Duration, + + entries: HashMap, + + children: HashMap>, + + /// The generation of the filesystem + /// This is used to invalidate the cache + /// when the filesystem is remounted + generation: u64, +} + +impl DriveFilesystem { + pub async fn new(root: impl AsRef) -> Result { + debug!("new: {:?};", root.as_ref()); + let mut entries = HashMap::new(); + let now = SystemTime::now(); + // Add root directory with inode number 1 + let root_attr = FileAttr { + ino: FUSE_ROOT_ID, + size: 0, + blocks: 0, + atime: now, + mtime: now, + ctime: now, + crtime: now, + kind: FileType::Directory, + perm: 0o755, + nlink: 2, + uid: 0, + gid: 0, + rdev: 0, + blksize: 4096, + flags: 0, + }; + let inode = FUSE_ROOT_ID.into(); + entries.insert( + inode, + DriveEntry { + ino: inode, + name: "root".into(), + local_path: LocalPath::from(Path::new("")), + drive_id: DriveId::root(), + // drive_path: "/".into(), + attr: root_attr, + }, + ); + + let cache_dir = tempfile::tempdir()?; + debug!("cache_dir: {:?}", cache_dir.path()); + if !cache_dir.path().exists() { + debug!("creating cache dir: {:?}", cache_dir.path()); + std::fs::create_dir_all(cache_dir.path())?; + } else { + debug!("cache dir exists: {}", cache_dir.path().display()); + } + let mut s = Self { + root: root.as_ref().to_path_buf(), + source: GoogleDrive::new().await?, + cache_dir: Some(cache_dir), + time_to_live: Duration::from_secs(2), + entries, + /*TODO: implement a way to increase this if necessary*/ + generation: 0, + children: HashMap::new(), + }; + // + // let root = s.root.to_path_buf(); + // s.add_dir_entry(&root, Inode::from(FUSE_ROOT_ID), true) + // .await; + + Ok(s) + } + fn get_cache_dir_for_file(&self, inode: Inode) -> Result { + debug!("get_cache_dir_for_file: {}", inode); + let cache_dir = self.cache_dir.as_ref().ok_or(anyhow!("no cache dir"))?; + debug!( + "get_cache_dir_for_file: {}, cache_dir: {}", + inode, + cache_dir.path().display() + ); + let entry = self + .entries + .get(&inode) + .ok_or(anyhow!("could not get entry"))?; + debug!( + "get_cache_dir_for_file: entry local_path: {}", + entry.local_path.display() + ); + let folder_path = match entry.local_path.parent() { + Some(p) => p.as_os_str(), + None => OsStr::new(""), + }; + debug!("get_cache_dir_for_file: folder_path: {:?}", folder_path); + let path = cache_dir.path().join(folder_path); + debug!("get_cache_dir_for_file: {}: {}", inode, path.display()); + Ok(path) + } + #[async_recursion::async_recursion] + async fn add_dir_entry( + &mut self, + folder_path: &Path, + parent_ino: impl Into + Send + 'async_recursion, + skip_self: bool, + ) -> Result<()> { + let parent_ino: Inode = parent_ino.into(); + let ino; + debug!( + "add_dir_entry: {:?}; parent: {}; skip_self: {} ", + folder_path, parent_ino, skip_self + ); + if self.root == folder_path { + ino = parent_ino; + } else { + debug!("add_dir_entry: adding entry for {:?}", folder_path); + ino = self + .add_entry( + folder_path.file_name().ok_or(anyhow!("invalid filename"))?, + /*TODO: correct permissions*/ + 0o755, + FileType::Directory, + parent_ino, + /*TODO: implement size for folders*/ 0, + ) + .await?; + } + + let drive = &self.source; + + let folder_drive_id: DriveId = self + .get_drive_id(ino) + .ok_or(anyhow!("could not find dir drive_id"))?; + debug!( + "add_dir_entry: getting files for '{:50?}' {}", + folder_drive_id, + folder_path.display() + ); + let files; + { + let files_res = self.source.list_files(folder_drive_id).await; + if let Err(e) = files_res { + warn!("could not get files: {}", e); + return Ok(()); + } + files = files_res.unwrap(); + } + debug!("got {} files", files.len()); + // let d = std::fs::read_dir(folder_path); + + for entry in files { + debug!("entry: {:?}", entry); + let name = entry.name.as_ref().ok_or_else(|| "no name"); + if let Err(e) = name { + warn!("could not get name: {}", e); + continue; + } + let name = name.as_ref().unwrap(); + if name.contains("/") || name.contains("\\") || name.contains(":") { + warn!("invalid name: {}", name); + continue; + } + let path = folder_path.join(&name); + + if let None = &entry.mime_type { + warn!("could not get mime_type"); + continue; + } + + let mime_type = entry.mime_type.as_ref().unwrap(); + if mime_type == "application/vnd.google-apps.document" + || mime_type == "application/vnd.google-apps.spreadsheet" + || mime_type == "application/vnd.google-apps.drawing" + || mime_type == "application/vnd.google-apps.form" + || mime_type == "application/vnd.google-apps.presentation" + || mime_type == "application/vnd.google-apps.drive-sdk" + || mime_type == "application/vnd.google-apps.script" + //TODO: add all relevant mime types + { + debug!( + "skipping google file: mime_type: '{}' entry: {:?}", + mime_type, entry + ); + continue; + } else if mime_type == "application/vnd.google-apps.folder" { + debug!("adding folder: {:?}", path); + let res = self.add_dir_entry(&path, ino, false).await; + if let Err(e) = res { + warn!("could not add folder: {}", e); + continue; + } + // } else if metadata.is_file() { + } else { + debug!("adding file: '{}' {:?}", mime_type, path); + let size = match Self::get_size_from_drive_metadata(&entry) { + Some(value) => value, + None => continue, + }; + let mode = 0o644; //TODO: get mode from settings + self.add_file_entry(ino, &OsString::from(&name), mode as u16, size) + .await; + } + } + + Ok(()) + } + + fn get_size_from_drive_metadata(entry: &File) -> Option { + let size = entry.size.ok_or_else(|| 0); + if let Err(e) = size { + warn!("could not get size: {}", e); + return None; + } + let size = size.unwrap(); + if size < 0 { + warn!("invalid size: {}", size); + return None; + } + let size = size as u64; + Some(size) + } + fn get_drive_id(&self, ino: impl Into) -> Option { + self.get_entry(ino).map(|e| e.drive_id.clone()) + } + async fn download_file_to_cache(&self, ino: impl Into) -> Result { + let ino = ino.into(); + debug!("download_file_to_cache: {}", ino); + let entry = self.get_entry_r(ino)?; + let drive_id = entry.drive_id.clone(); + let drive = &self.source; + let path = self.get_cache_path_for_entry(&entry)?; + let folder = path.parent().unwrap(); + if !folder.exists() { + debug!("creating folder: {}", folder.display()); + std::fs::create_dir_all(folder)?; + } + debug!("downloading file: {}", path.display()); + drive.download_file(drive_id, &path).await?; + Ok(path) + } + fn check_if_file_is_cached(&self, ino: impl Into) -> Result { + let entry = self.get_entry_r(ino)?; + let path = self.get_cache_path_for_entry(&entry)?; + let exists = path.exists(); + Ok(exists) + } + + fn get_cache_path_for_entry(&self, entry: &&DriveEntry) -> Result { + debug!("get_cache_path_for_entry: {}", entry.ino); + let folder = self.get_cache_dir_for_file(entry.ino)?; + let path = folder.join(&entry.name); + debug!( + "get_cache_path_for_entry: {}: {}", + entry.ino, + path.display() + ); + Ok(path) + } +} +#[async_trait::async_trait] +impl CommonFilesystem for DriveFilesystem { + fn get_entries(&self) -> &HashMap { + &self.entries + } + + fn get_entries_mut(&mut self) -> &mut HashMap { + &mut self.entries + } + + fn get_children(&self) -> &HashMap> { + &self.children + } + + fn get_children_mut(&mut self) -> &mut HashMap> { + &mut self.children + } + + fn get_root_path(&self) -> LocalPath { + self.root.clone().into() + } + + async fn add_entry( + &mut self, + name: &OsStr, + mode: u16, + file_type: FileType, + parent_ino: impl Into + Send, + size: u64, + ) -> Result { + let parent_ino = parent_ino.into(); + debug!("add_entry: (0) name:{:20?}; parent: {}", name, parent_ino); + let ino = self.generate_ino(); // Generate a new inode number + let now = std::time::SystemTime::now(); + let attr = FileAttr { + ino: ino.into(), + size: size, + blocks: 0, + atime: now, + mtime: now, + ctime: now, + crtime: now, + kind: file_type, + perm: mode, + nlink: 1, + uid: 0, + gid: 0, + rdev: 0, + blksize: 4096, + flags: 0, + }; + + let parent_drive_id = self.get_drive_id(parent_ino); + let drive_id: DriveId = self.source.get_id(name, parent_drive_id).await?; + debug!("add_entry: (1) drive_id: {:?}", drive_id); + + let parent_local_path = self.get_path_from_ino(parent_ino); + let parent_path: PathBuf = parent_local_path + .ok_or(anyhow!("could not get local path"))? + .into(); + + self.get_entries_mut().insert( + ino, + DriveEntry::new(ino, name, drive_id, parent_path.join(name), attr), + ); + + self.add_child(parent_ino, &ino); + debug!("add_entry: (2) after adding count: {}", self.entries.len()); + Ok(ino) + } +} + +impl Display for DriveFilesystem { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "DriveFilesystem at: '/{}'", self.root.display()) + } +} +impl Filesystem for DriveFilesystem { + fn init( + &mut self, + _req: &Request<'_>, + _config: &mut KernelConfig, + ) -> std::result::Result<(), c_int> { + debug!("init"); + + let root = self.root.to_path_buf(); + let x = run_async_blocking(self.add_dir_entry(&root, Inode::from(FUSE_ROOT_ID), true)); + if let Err(e) = x { + error!("could not add root entry: {}", e); + } + Ok(()) + } + fn destroy(&mut self) { + debug!("destroy"); + debug!("destroy: removing cache dir: {:?}", self.cache_dir); + self.cache_dir = None; + } + fn lookup(&mut self, _req: &Request<'_>, parent: u64, name: &OsStr, reply: ReplyEntry) { + debug!("lookup: {}:{:?}", parent, name); + let parent = parent.into(); + let children = self.children.get(&parent); + if children.is_none() { + warn!("lookup: could not find children for {}", parent); + reply.error(libc::ENOENT); + return; + } + let children = children.unwrap(); + debug!("lookup: children: {:?}", children); + for child_inode in children { + let entry = self.entries.get(child_inode); + if entry.is_none() { + warn!("lookup: could not find entry for {}", child_inode); + continue; + } + let entry = entry.unwrap(); + + let path: PathBuf = entry.name.clone().into(); + let accepted = name.eq_ignore_ascii_case(&path); + debug!( + "entry: {}:(accepted={}){:?}; {:?}", + child_inode, accepted, path, entry.attr + ); + if accepted { + reply.entry(&self.time_to_live, &entry.attr, self.generation); + return; + } + } + warn!("lookup: could not find entry for {:?}", name); + + reply.error(libc::ENOENT); + } + fn getattr(&mut self, _req: &Request<'_>, ino: u64, reply: ReplyAttr) { + debug!("getattr: {}", ino); + let entry = self.entries.get(&ino.into()); + if let Some(entry) = entry { + reply.attr(&self.time_to_live, &entry.attr); + } else { + reply.error(libc::ENOENT); + } + } + + fn read( + &mut self, + _req: &Request<'_>, + ino: u64, + fh: u64, + offset: i64, + size: u32, + flags: i32, + lock_owner: Option, + reply: ReplyData, + ) { + debug!( + "read: {:10}:{:2}:{:3}:{:10}:{:10X}:{:?}", + ino, fh, offset, size, flags, lock_owner + ); + + let entry = self.get_entry_r(&ino.into()); + if let Err(e) = entry { + error!("read: could not find entry for {}: {}", ino, e); + reply.error(libc::ENOENT); + return; + } + let entry = entry.unwrap(); + + let is_cached = self.check_if_file_is_cached(ino); + if !is_cached.unwrap_or(false) { + debug!("read: file is not cached: {}", ino); + let x: Result = run_async_blocking(self.download_file_to_cache(ino)); + + if let Err(e) = x { + error!("read: could not download file: {}", e); + reply.error(libc::ENOENT); + return; + } + } + + let path = self.get_cache_path_for_entry(&entry); + if let Err(e) = path { + error!("read: could not get cache path: {}", e); + reply.error(libc::ENOENT); + return; + } + let path = path.unwrap(); + + debug!("read: path: {:?}", path); + let file = std::fs::File::open(&path); + if let Err(e) = file { + error!("read: could not open file: {}", e); + reply.error(libc::ENOENT); + return; + } + let mut file = file.unwrap(); + + let mut buf = vec![0; size as usize]; + debug!("reading file: {:?} at {} with size {}", &path, offset, size); + file.read_at(&mut buf, offset as u64).unwrap(); + debug!("read file: {:?} at {}", &path, offset); + reply.data(&buf); + } + fn readdir( + &mut self, + _req: &Request<'_>, + ino: u64, + fh: u64, + mut offset: i64, + mut reply: ReplyDirectory, + ) { + debug!("readdir: {}:{}:{:?}", ino, fh, offset); + let children = self.children.get(&ino.into()); + if let Some(attr) = self.get_entries().get(&ino.into()).map(|entry| entry.attr) { + if attr.kind != FileType::Directory { + reply.error(libc::ENOTDIR); + return; + } + } + if !(children.is_none()) { + } else { + reply.error(libc::ENOENT); + return; + } + + let children = children.unwrap(); + debug!("children ({}): {:?}", children.len(), children); + for child_inode in children.iter().skip(offset as usize) { + let entry = self.entries.get(child_inode).unwrap(); + let path: PathBuf = entry.local_path.clone().into(); + let attr = entry.attr; + let inode = (*child_inode).into(); + offset += 1; // Increment the offset for each processed entry + debug!("entry: {}:{:?}; {:?}", inode, path, attr); + if reply.add(inode, offset, attr.kind, &entry.name) { + // If the buffer is full, we need to stop + debug!("readdir: buffer full"); + break; + } + } + debug!("readdir: ok"); + reply.ok(); + } + fn access(&mut self, _req: &Request<'_>, ino: u64, mask: i32, reply: ReplyEmpty) { + reply.ok(); //TODO: implement this correctly + } +} diff --git a/src/fs/inode.rs b/src/fs/inode.rs new file mode 100644 index 0000000..81d0d60 --- /dev/null +++ b/src/fs/inode.rs @@ -0,0 +1,56 @@ +use std::fmt::Display; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct Inode(u64); + +impl Inode { + pub fn new(value: u64) -> Self { + Self(value) + } + pub fn get(&self) -> u64 { + self.0 + } +} + +impl Display for Inode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl Into for Inode { + fn into(self) -> u64 { + self.0 + } +} + +// impl Into for Inode{ +// fn into(self) -> Inode { +// self +// } +// } + +impl TryInto for Inode { + type Error = std::num::TryFromIntError; + + fn try_into(self) -> Result { + self.0.try_into() + } +} + +impl From for Inode { + fn from(value: u64) -> Inode { + Inode(value) + } +} +impl From for Inode { + fn from(value: u32) -> Inode { + Inode(value as u64) + } +} + +impl From<&Inode> for Inode { + fn from(value: &Inode) -> Self { + value.clone() + } +} diff --git a/src/fs/mod.rs b/src/fs/mod.rs new file mode 100644 index 0000000..d4e2b4b --- /dev/null +++ b/src/fs/mod.rs @@ -0,0 +1,6 @@ +mod common; +mod inode; +pub use common::*; +pub use inode::*; +pub mod drive; +pub mod sample; diff --git a/src/fs/sample.rs b/src/fs/sample.rs new file mode 100644 index 0000000..84d7c21 --- /dev/null +++ b/src/fs/sample.rs @@ -0,0 +1,474 @@ +// use crate::async_helper::run_async_in_sync; +use crate::async_helper::run_async_blocking; +use crate::common::LocalPath; +use crate::fs::common::CommonFilesystem; +use crate::fs::inode::Inode; +use crate::fs::CommonEntry; +use crate::prelude::*; +use fuser::{ + FileAttr, FileType, KernelConfig, ReplyAttr, ReplyData, ReplyDirectory, ReplyEmpty, ReplyEntry, + ReplyOpen, ReplyStatfs, ReplyWrite, ReplyXattr, Request, TimeOrNow, FUSE_ROOT_ID, +}; +use libc::c_int; +use log::{debug, warn}; +use std::collections::HashMap; +use std::ffi::{OsStr, OsString}; +use std::fmt::Display; +use std::fs::OpenOptions; +use std::os::unix::prelude::*; +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +#[derive(Debug)] +struct SampleEntry { + pub ino: Inode, + + pub name: OsString, + pub local_path: LocalPath, + pub attr: FileAttr, +} + +impl SampleEntry { + // fn new(ino: impl Into, local_path: OsString, attr: FileAttr) -> Self { + // Self { + // ino: ino.into(), + // name: OsString::new(), + // local_path: LocalPath::from(Path::new(&local_path)), + // attr, + // } + // } + + fn new( + ino: impl Into, + name: impl Into, + local_path: impl Into, + attr: FileAttr, + ) -> Self { + Self { + ino: ino.into(), + name: name.into(), + local_path: local_path.into(), + attr, + } + } +} + +impl CommonEntry for SampleEntry { + fn get_ino(&self) -> Inode { + self.ino + } + + fn get_name(&self) -> &OsStr { + self.name.as_os_str() + } + + fn get_local_path(&self) -> &LocalPath { + &self.local_path + } + + fn get_attr(&self) -> &FileAttr { + &self.attr + } +} +#[derive(Debug, Default)] +pub struct SampleFilesystem { + /// the point where the filesystem is mounted + root: PathBuf, + /// the source dir to read from and write to + source: PathBuf, + + /// How long the responses can/should be cached + time_to_live: Duration, + + entries: HashMap, + + children: HashMap>, + + /// The generation of the filesystem + /// This is used to invalidate the cache + /// when the filesystem is remounted + generation: u64, +} +impl SampleFilesystem { + pub fn new(root: impl AsRef, source: impl AsRef) -> Self { + debug!("new: {:?}; {:?}", root.as_ref(), source.as_ref()); + let mut entries = HashMap::new(); + // Add root directory with inode number 1 + let root_attr = FileAttr { + ino: FUSE_ROOT_ID, + size: 0, + blocks: 0, + atime: UNIX_EPOCH, + mtime: UNIX_EPOCH, + ctime: UNIX_EPOCH, + crtime: UNIX_EPOCH, + kind: FileType::Directory, + perm: 0o755, + nlink: 2, + uid: 0, + gid: 0, + rdev: 0, + blksize: 4096, + flags: 0, + }; + entries.insert( + FUSE_ROOT_ID.into(), + SampleEntry::new( + FUSE_ROOT_ID, + "root", + LocalPath::from(root.as_ref()), + root_attr, + ), + ); + + Self { + root: root.as_ref().to_path_buf(), + source: source.as_ref().to_path_buf(), + time_to_live: Duration::from_secs(2), + entries, + /*TODO: implement a way to increase this if necessary*/ + generation: 0, + children: HashMap::new(), + } + } +} +#[async_trait::async_trait] +impl CommonFilesystem for SampleFilesystem { + fn get_entries(&self) -> &HashMap { + &self.entries + } + fn get_entries_mut(&mut self) -> &mut HashMap { + &mut self.entries + } + fn get_children(&self) -> &HashMap> { + &self.children + } + fn get_children_mut(&mut self) -> &mut HashMap> { + &mut self.children + } + fn get_root_path(&self) -> LocalPath { + self.source.clone().into() + } + async fn add_entry( + &mut self, + name: &OsStr, + mode: u16, + file_type: FileType, + parent_ino: impl Into + Send, + size: u64, + ) -> Result { + let parent_ino = parent_ino.into(); + let ino = self.generate_ino(); // Generate a new inode number + let now = std::time::SystemTime::now(); + let attr = FileAttr { + ino: ino.into(), + size: size, + blocks: 0, + atime: now, + mtime: now, + ctime: now, + crtime: now, + kind: file_type, + perm: mode, + nlink: 1, + uid: 0, + gid: 0, + rdev: 0, + blksize: 4096, + flags: 0, + }; + + self.get_entries_mut() + .insert(ino, SampleEntry::new(ino, name, OsString::from(name), attr)); + + self.add_child(parent_ino, &ino); + Ok(ino) + } +} +impl SampleFilesystem { + async fn add_dir_entry( + &mut self, + folder_path: &Path, + parent_ino: impl Into, + skip_self: bool, + ) -> Result<()> { + let parent_ino = parent_ino.into(); + let ino: Inode; + if skip_self { + ino = parent_ino; + } else { + ino = self + .add_entry( + folder_path.file_name().unwrap(), + /*TODO: correct permissions*/ + 0o755, + FileType::Directory, + parent_ino, + /*TODO: implement size for folders*/ 0, + ) + .await?; + } + let d = std::fs::read_dir(folder_path); + if let Ok(d) = d { + for entry in d { + if let Ok(entry) = entry { + let path = entry.path(); + let name = entry.file_name(); + let metadata = entry.metadata(); + if let Ok(metadata) = metadata { + if metadata.is_dir() { + self.add_dir_entry(&path, ino, false); + } else if metadata.is_file() { + let mode = metadata.mode(); + let size = metadata.size(); + //TODO: async call + // self.add_file_entry(ino, name.as_os_str(), mode as u16, size); + } + } + } + } + } + Ok(()) + } +} + +impl fuser::Filesystem for SampleFilesystem { + fn init( + &mut self, + _req: &Request<'_>, + _config: &mut KernelConfig, + ) -> std::result::Result<(), c_int> { + debug!("init"); + // self.add_file_entry(1, "hello.txt".as_ref(), 0o644); + let source = self.source.clone(); + + run_async_blocking(async { + self.add_dir_entry(&source, FUSE_ROOT_ID, true).await; + }); + // self.add_dir_entry(&source, FUSE_ROOT_ID, true); + Ok(()) + } + fn lookup(&mut self, _req: &Request<'_>, parent: u64, name: &OsStr, reply: ReplyEntry) { + debug!("lookup: {}:{:?}", parent, name); + for (inode, entry) in self.entries.iter() { + let path: PathBuf = entry.local_path.clone().into(); + let accepted = name.eq_ignore_ascii_case(&path); + debug!( + "entry: {}:(accepted={}){:?}; {:?}", + inode, accepted, path, entry.attr + ); + if accepted { + reply.entry(&self.time_to_live, &entry.attr, self.generation); + return; + } + } + + reply.error(libc::ENOENT); + } + fn getattr(&mut self, _req: &Request<'_>, ino: u64, reply: ReplyAttr) { + self.entries.get(&ino.into()).map(|entry| { + reply.attr(&self.time_to_live, &entry.attr); + }); + } + fn readdir( + &mut self, + _req: &Request<'_>, + ino: u64, + fh: u64, + mut offset: i64, + mut reply: ReplyDirectory, + ) { + debug!("readdir: {}:{}:{:?}", ino, fh, offset); + let children = self.children.get(&ino.into()); + if let Some(attr) = self.get_entries().get(&ino.into()).map(|entry| entry.attr) { + if attr.kind != FileType::Directory { + reply.error(libc::ENOTDIR); + return; + } + } + if !(children.is_none()) { + } else { + reply.error(libc::ENOENT); + return; + } + + let children = children.unwrap(); + debug!("children ({}): {:?}", children.len(), children); + for child_inode in children.iter().skip(offset as usize) { + let entry = self.entries.get(child_inode).unwrap(); + let path: PathBuf = entry.local_path.clone().into(); + let attr = entry.attr; + let inode = (*child_inode).into(); + offset += 1; // Increment the offset for each processed entry + debug!("entry: {}:{:?}; {:?}", inode, path, attr); + if !reply.add(inode, offset, attr.kind, path) { + break; + } + } + debug!("readdir: ok"); + reply.ok(); + } + fn read( + &mut self, + _req: &Request<'_>, + ino: u64, + fh: u64, + offset: i64, + size: u32, + flags: i32, + lock_owner: Option, + reply: ReplyData, + ) { + debug!( + "read: {}:{}:{}:{}:{:#x?}:{:?}", + ino, fh, offset, size, flags, lock_owner + ); + let data = self.get_entry(ino).map(|entry| entry.attr); + if let Some(attr) = data { + if attr.kind != FileType::RegularFile { + reply.error(libc::EISDIR); + return; + } + + let path = self.get_full_path_from_ino(ino); + debug!("opening file: {:?}", &path); + let mut file = std::fs::File::open::(path.clone().unwrap().into()).unwrap(); + let mut buf = vec![0; size as usize]; + debug!("reading file: {:?} at {} with size {}", &path, offset, size); + file.read_at(&mut buf, offset as u64).unwrap(); + debug!("read file: {:?} at {}", &path, offset); + reply.data(&buf); + } else { + reply.error(libc::ENOENT); + } + } + fn write( + &mut self, + _req: &Request<'_>, + ino: u64, + fh: u64, + offset: i64, + data: &[u8], + write_flags: u32, + flags: i32, + lock_owner: Option, + reply: ReplyWrite, + ) { + debug!( + "write: {}:{}:{}:{:#x?}:{:?}:{:#x?}:{:?}", + ino, fh, offset, flags, lock_owner, write_flags, data, + ); + let attr = self.get_entry(ino).map(|entry| entry.attr); + if let Some(attr) = attr { + if attr.kind != FileType::RegularFile { + warn!( + "write: not a file, writing is not supported: kind:{:?}; attr:{:?}", + attr.kind, attr + ); + reply.error(libc::EISDIR); + return; + } + + let path = self.get_full_path_from_ino(ino); + debug!("opening file: {:?}", &path); + let mut file = OpenOptions::new() + .write(true) + .create(true) + .open::(path.clone().unwrap().into()) + .unwrap(); + debug!( + "writing file: {:?} at {} with size {}", + &path, + offset, + data.len() + ); + + let size = file.write_at(data, offset as u64).unwrap(); + debug!("wrote file: {:?} at {}; wrote {} bits", &path, offset, size); + reply.written(size as u32); + } else { + reply.error(libc::ENOENT); + } + } + + fn setattr( + &mut self, + _req: &Request<'_>, + ino: u64, + mode: Option, + uid: Option, + gid: Option, + size: Option, + _atime: Option, + _mtime: Option, + _ctime: Option, + /*TODO: check if this change need to be implemented*/ + fh: Option, + _crtime: Option, + /*TODO: check if this change need to be implemented*/ + _chgtime: Option, + /*TODO: check if this change need to be implemented*/ + _bkuptime: Option, + flags: Option, + reply: ReplyAttr, + ) { + debug!( + "setattr: {}:{:?}:{:?}:{:?}:{:?}:{:?}:{:?}:{:?}:{:?}:{:?}:{:?}:{:?}:{:?}", + ino, + mode, + uid, + gid, + size, + _atime, + _mtime, + _ctime, + fh, + _crtime, + _chgtime, + _bkuptime, + flags + ); + let attr = self + .entries + .get_mut(&ino.into()) + .map(|entry| &mut entry.attr); + if attr.is_none() { + reply.error(libc::ENOENT); + return; + } + let mut attr = attr.unwrap(); + + if let Some(mode) = mode { + attr.perm = mode as u16; + } + if let Some(uid) = uid { + attr.uid = uid; + } + if let Some(gid) = gid { + attr.gid = gid; + } + if let Some(size) = size { + attr.size = size; + } + if let Some(atime) = _atime { + attr.atime = Self::convert_to_system_time(atime); + } + if let Some(mtime) = _mtime { + attr.mtime = Self::convert_to_system_time(mtime); + } + if let Some(ctime) = _ctime { + attr.ctime = ctime; + } + if let Some(crtime) = _crtime { + attr.crtime = crtime; + } + if let Some(flags) = flags { + attr.flags = flags; + } + + reply.attr(&self.time_to_live, attr); + } + fn access(&mut self, _req: &Request<'_>, ino: u64, mask: i32, reply: ReplyEmpty) { + reply.ok(); //TODO: implement this a bit better/more useful + } +} diff --git a/src/google_drive/drive.rs b/src/google_drive/drive.rs new file mode 100644 index 0000000..59f4464 --- /dev/null +++ b/src/google_drive/drive.rs @@ -0,0 +1,367 @@ +use crate::google_drive::{drive, helpers, DriveId}; +use crate::prelude::*; +use std::ffi::{OsStr, OsString}; +// use drive3::api::Scope::File; +use anyhow::anyhow; +use drive3::api::{File, Scope}; +use drive3::client::ReadSeek; +use drive3::hyper::body::HttpBody; +use drive3::hyper::client::HttpConnector; +use drive3::hyper::{body, Body, Response}; +use drive3::hyper_rustls::HttpsConnector; +use drive3::DriveHub; +use drive3::{hyper_rustls, oauth2}; +use futures::{Stream, StreamExt}; +use hyper::Client; +use log::{debug, trace, warn}; +use mime::{FromStrError, Mime}; +use std::fmt::{Debug, Error}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; +use tokio::runtime::Runtime; +use tokio::{fs, io}; + +pub struct GoogleDrive { + hub: DriveHub>, +} + +impl GoogleDrive { + pub async fn download_file(&self, file_id: DriveId, target_file: &PathBuf) -> Result<()> { + debug!( + "download_file: file_id: {:50?} to {}", + file_id, + target_file.display() + ); + let file_id: String = match file_id.try_into() { + Ok(file_id) => file_id, + Err(e) => return Err(anyhow!("invalid file_id: {:?}", e).into()), + }; + + let x = download_file_by_id(&self, file_id, target_file.as_path()).await; + debug!("download_file: completed"); + let x = x?; + + debug!("download_file: success"); + + Ok(()) + } +} + +impl GoogleDrive { + pub async fn get_id(&self, path: &OsStr, parent_drive_id: Option) -> Result { + debug!("Get ID of '{:?}' with parent: {:?}", path, parent_drive_id); + let path: OsString = path.into(); + let path = match path.into_string() { + Ok(path) => path, + Err(_) => return Err("invalid path".into()), + }; + let parent_drive_id: OsString = match parent_drive_id { + Some(parent_drive_id) => parent_drive_id, + None => DriveId::from("root"), + } + .into(); + let parent_drive_id = match parent_drive_id.into_string() { + Ok(parent_drive_id) => parent_drive_id, + Err(_) => return Err("invalid parent_drive_id".into()), + }; + debug!("get_id: path: {}", path); + debug!("get_id: parent_drive_id: {}", parent_drive_id); + + let req = self + .hub + .files() + .list() + .q(&format!( + // "'{}' in parents, '{}' == name", + "name = '{}' and '{}' in parents", + path, parent_drive_id + )) + .param("fields", "files(id)") + .doit() + .await; + let (response, files) = match req { + Ok((response, files)) => (response, files), + Err(e) => { + warn!("get_id: Error: {}", e); + return Err("Error".into()); + } + }; + + if files.files.is_none() { + warn!("get_id: No files found (0)"); + return Err("No files found".into()); + } + let files = files.files.unwrap(); + if files.len() == 0 { + warn!("get_id: No files found (1)"); + return Err("No files found".into()); + } + if files.len() > 1 { + warn!("get_id: Multiple files found"); + return Err("Multiple files found".into()); + } + let file = files.into_iter().next().unwrap(); + let id = file.id.unwrap(); + debug!("get_id: id: {}", id); + Ok(DriveId::from(id)) + } +} + +impl GoogleDrive { + pub(crate) async fn new() -> Result { + let auth = drive3::oauth2::read_application_secret("auth/client_secret.json").await?; + + let auth = oauth2::InstalledFlowAuthenticator::builder( + auth, + oauth2::InstalledFlowReturnMethod::HTTPRedirect, + ) + .persist_tokens_to_disk("auth/tokens.json") + .build() + .await?; + let http_client = Client::builder().build( + hyper_rustls::HttpsConnectorBuilder::new() + .with_native_roots() + .https_or_http() + .enable_http1() + .enable_http2() + .build(), + ); + let hub = DriveHub::new(http_client, auth); + + let mut drive = GoogleDrive { hub }; + Ok(drive) + } + pub async fn list_files(&mut self, folder_id: DriveId) -> Result> { + let folder_id: OsString = folder_id.into(); + let folder_id = match folder_id.into_string() { + Ok(folder_id) => folder_id, + Err(_) => return Err("invalid folder_id".into()), + }; + if folder_id.is_empty() { + return Err("folder_id is empty".into()); + } + if folder_id.contains('\'') { + return Err("folder_id contains invalid character".into()); + } + let mut files = Vec::new(); + let mut page_token = None; + loop { + let (response, result) = self + .hub + .files() + .list() + .param( + "fields", + "nextPageToken, files(id, name, size, mimeType, kind)", + ) + // .page_token(page_token.as_ref().map(String::as_str)) + .q(format!("'{}' in parents", folder_id).as_str()) + .doit() + .await?; + files.extend(result.files.ok_or("no file list returned")?); + page_token = result.next_page_token; + if page_token.is_none() { + break; + } + } + Ok(files) + } +} +impl Debug for GoogleDrive { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "GoogleDrive") + } +} +pub async fn sample() -> Result<()> { + debug!("sample"); + + let mut drive = GoogleDrive::new().await?; + + sample_list_files(&mut drive).await?; + let hello_world_file = get_files_by_name(&mut drive, "hello_world.txt").await?; + let hello_world_file = hello_world_file + .first() + .ok_or("hello_world.txt not found")?; + debug!("hello_world_file: id:{:?}", hello_world_file.id); + let target_path = "/tmp/hello_world.txt"; + let target_path = std::path::Path::new(target_path); + // download_file(&mut drive, hello_world_file, target_path).await?; + debug!("target_path: {:?}", target_path); + debug!("download_file_by_id"); + let hello_world_file_id = hello_world_file.id.as_ref().ok_or("")?; + download_file_by_id(&mut drive, hello_world_file_id, target_path).await?; + debug!("get_file_header_by_id"); + get_file_header_by_id(&mut drive, hello_world_file_id).await?; + debug!("done"); + Ok(()) +} + +async fn download_file( + hub: &GoogleDrive, + file: &drive3::api::File, + target_path: &Path, +) -> Result { + if let Some(id) = &file.id { + download_file_by_id(hub, id, target_path).await + } else { + Err("file id not found".into()) + } +} + +async fn download_file_by_id( + hub: &GoogleDrive, + id: impl Into, + target_path: &Path, +) -> Result { + use tokio::fs::File; + use tokio::io::AsyncWriteExt; + let (response, content): (Response, google_drive3::api::File) = hub + .hub + .files() + .get(&id.into()) + .add_scope(Scope::Readonly) + .acknowledge_abuse(true) + .param("alt", "media") + .doit() + .await?; + //TODO: bigger files don't get downloaded. it just starts and then hangs at ~1.3MB forever + debug!("download_file_by_id(): response: {:?}", response); + debug!("download_file_by_id(): content: {:?}", content); + write_body_to_file(response, target_path).await?; + + Ok(content) +} + +async fn write_body_to_file(response: Response, target_path: &Path) -> Result<()> { + use futures::StreamExt; + debug!("write_body_to_file(): target_path: {:?}", target_path); + + let mut file = std::fs::File::create(target_path)?; + + let mut stream = response.into_body(); + let mut buffer = bytes::BytesMut::new(); + let mut counter = 0; + while let Some(chunk) = stream.next().await { + let chunk = chunk?; + trace!("write_body_to_file(): chunk counter: {}", counter); + file.write_all(&chunk)?; + counter += 1; + } + debug!("write_body_to_file(): done"); + Ok(()) +} + +async fn get_file_header_by_id(hub: &GoogleDrive, id: &str) -> Result { + debug!("get_file_header_by_id(): id: {:?}", id); + let (response, content) = hub.hub.files().get(id).doit().await?; + + Ok(content) +} + +async fn get_files_by_name( + drive: &GoogleDrive, + name: impl Into, +) -> Result> { + let name = name.into(); + if name.is_empty() { + return Err("name cannot be empty".into()); + } + if name.contains("'") { + return Err("name cannot contain single quote".into()); + } + let (response, files) = drive + .hub + .files() + .list() + .q(format!("name = '{}'", name).as_str()) + .doit() + .await?; + debug!("get_files_by_name(): response: {:?}", response); + debug!("get_files_by_name(): files: {:?}", files); + let files: Vec = files.files.unwrap_or(vec![]); + Ok(files) +} + +async fn sample_list_files(drive: &GoogleDrive) -> Result<()> { + let (hello_world_res, hello_world_list) = drive + .hub + .files() + .list() + // .q("name = 'hello_world.txt'") + // .q("'root' in parents and trashed=false") + .doit() + .await?; + debug!("hello_world_res: {:?}", hello_world_res); + debug!("hello_world_list: {:?}", hello_world_list); + let files: Vec = hello_world_list.files.unwrap_or(vec![]); + debug!("hello_world_list amount of files: {}", files.len()); + for file in files { + let name = file.name.unwrap_or("NO NAME".to_string()); + let id = file.id.unwrap_or("NO ID".to_string()); + let kind = file.kind.unwrap_or("NO KIND".to_string()); + let mime_type = file.mime_type.unwrap_or("NO MIME TYPE".to_string()); + + debug!( + "file: {:100}name:{:100}kind: {:25}mime_type: {:100}", + id, name, kind, mime_type + ); + } + + Ok(()) +} + +async fn create_file_on_drive_from_path( + drive: &GoogleDrive, + file: File, + path: &Path, + mime_type: mime::Mime, +) -> Result<()> { + let content = fs::File::open(path).await?; + create_file_on_drive(drive, file, mime_type, content).await?; + Ok(()) +} + +async fn create_file_on_drive( + drive: &GoogleDrive, + file: google_drive3::api::File, + mime_type: mime::Mime, + content: tokio::fs::File, +) -> Result { + let stream = content.into_std().await; + let (response, file) = drive + .hub + .files() + .create(file) + .upload_resumable(stream, mime_type) + .await?; + debug!("create_file(): response: {:?}", response); + debug!("create_file(): file: {:?}", file); + Ok(file) +} +pub async fn update_file_content_on_drive_from_path( + drive: &GoogleDrive, + file: google_drive3::api::File, + source_path: &Path, +) -> Result<()> { + Ok(()) +} +async fn update_file_content_on_drive( + drive: &GoogleDrive, + file: google_drive3::api::File, + content: fs::File, +) -> Result<()> { + let stream = content.into_std().await; + let mime_type = helpers::get_mime_from_file_metadata(&file)?; + let id = file.id.clone().unwrap(); + let (response, file) = drive + .hub + .files() + .update(file, &id) + .upload(stream, mime_type) + .await?; + debug!("update_file_on_drive(): response: {:?}", response); + debug!("update_file_on_drive(): file: {:?}", file); + Ok(()) +} diff --git a/src/google_drive/drive_id.rs b/src/google_drive/drive_id.rs new file mode 100644 index 0000000..b4f752b --- /dev/null +++ b/src/google_drive/drive_id.rs @@ -0,0 +1,43 @@ +use std::ffi::OsString; +use std::fmt::Display; +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct DriveId(OsString); + +impl DriveId { + pub(crate) fn root() -> DriveId { + DriveId(OsString::from("root")) + } +} + +impl Into for DriveId { + fn into(self) -> OsString { + self.0 + } +} +impl TryInto for DriveId { + type Error = OsString; + + fn try_into(self) -> Result { + self.0.into_string() + } +} +impl From for DriveId { + fn from(value: OsString) -> Self { + DriveId(value) + } +} +impl From for DriveId { + fn from(value: String) -> Self { + OsString::from(value).into() + } +} +impl From<&str> for DriveId { + fn from(s: &str) -> Self { + DriveId(OsString::from(s)) + } +} +impl DriveId { + pub fn new(id: impl Into) -> Self { + Self(id.into()) + } +} diff --git a/src/google_drive/helpers.rs b/src/google_drive/helpers.rs new file mode 100644 index 0000000..ebc72c4 --- /dev/null +++ b/src/google_drive/helpers.rs @@ -0,0 +1,69 @@ +use crate::fs::drive::DriveFilesystem; +use crate::fs::{CommonFilesystem, Inode}; +use crate::google_drive::{DriveId, GoogleDrive}; +use crate::prelude::*; +use anyhow::anyhow; +use drive3::api::File; +use log::debug; +use mime::Mime; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +pub fn get_mime_from_file_metadata(file: &File) -> Result { + Ok(Mime::from_str( + &file.mime_type.as_ref().unwrap_or(&"*/*".to_string()), + )?) +} +pub fn get_drive_id_from_local_path(drive: &DriveFilesystem, path: &Path) -> Result { + let drive_mount_point: &PathBuf = &drive.get_root_path().into(); + debug!("get_drive_id_from_path(): (0) path: '{}'", path.display()); + let path = match path.strip_prefix(drive_mount_point) { + Err(e) => { + return Err(anyhow!( + "Path {:?} is not a prefix of {:?}", + drive_mount_point, + path + ))? + } + Ok(path) => path, + }; + debug!("get_drive_id_from_path(): (1) path: '{}'", path.display()); + if path == Path::new("/") || path == Path::new("") { + debug!( + "get_drive_id_from_path(): (1) path is root: '{}'", + path.display() + ); + return Ok("root".into()); + } + + let mut parent_ino: Inode = 5u32.into(); + // let mut parent_ino : Inode =Inode::from(5u32);//.into(); + for part in path.iter() { + debug!("get_drive_id_from_path(): (2..) path: '{:?}'", part); + + let children = drive.get_children().get(&parent_ino); + debug!("get_drive_id_from_path(): (2..) children: '{:?}'", children); + } + todo!("get_drive_id_from_path()") +} +mod test { + use super::*; + #[tokio::test] + async fn test_get_drive_id_from_local_path() { + crate::init_logger(); + let path = Path::new("/drive1"); + let drive = DriveFilesystem::new(path).await; + let drive_mount_point = Path::new("/drive1"); + + let drive_id = get_drive_id_from_local_path(&drive, path).unwrap(); + assert_eq!(drive_id, "root".into()); + + let path = Path::new("/drive1/"); + let drive_id = get_drive_id_from_local_path(&drive, path).unwrap(); + assert_eq!(drive_id, "root".into()); + + let path = Path::new("/drive1/dir1/dir2/file1.txt"); + let drive_id = get_drive_id_from_local_path(&drive, path).unwrap(); + todo!("create assert for this test"); + // assert_eq!(drive_id, "TODO".into()); + } +} diff --git a/src/google_drive/mod.rs b/src/google_drive/mod.rs new file mode 100644 index 0000000..5f6c992 --- /dev/null +++ b/src/google_drive/mod.rs @@ -0,0 +1,8 @@ +mod helpers; +pub use drive::*; +pub use helpers::*; + +mod drive; + +mod drive_id; +pub use drive_id::*; diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..c44a1d3 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,476 @@ +#![allow(dead_code, unused)] + +use google_drive3::oauth2::read_application_secret; +use std::error::Error; +use std::time::{Duration, SystemTime}; + +extern crate google_drive3 as drive3; +use drive3::api::Channel; +use drive3::{hyper, hyper_rustls, oauth2, DriveHub}; +use log::{debug, error, info, trace, warn}; +// use nix; +use notify::{recommended_watcher, INotifyWatcher, RecommendedWatcher}; +use tokio::io::{stdin, AsyncReadExt}; +use tokio::sync::mpsc::channel; +pub mod async_helper; +pub mod common; +pub mod fs; +pub mod google_drive; +pub mod prelude; + +use prelude::*; + +#[cfg(test)] +mod tests { + use super::*; + fn init_logger() { + let _ = env_logger::builder() + .filter_level(log::LevelFilter::Trace) + .is_test(true) + .try_init(); + } + + #[tokio::test] + async fn does_it_work() { + init_logger(); + list_files().await; + } +} + +pub fn init_logger() { + env_logger::builder() + .filter_level(log::LevelFilter::Debug) + .try_init(); +} + +pub async fn sample() -> Result<()> { + //Test file id: "1IotISYu3cF7JrOdfFPKNOkgYg1-ii5Qs" + list_files().await +} +async fn list_files() -> Result<()> { + debug!("Hello, world!"); + let secret: oauth2::ApplicationSecret = read_application_secret("auth/client_secret.json") + .await + .expect("failed to read client secret file"); + let auth = oauth2::InstalledFlowAuthenticator::builder( + secret, + oauth2::InstalledFlowReturnMethod::HTTPRedirect, + ) + .persist_tokens_to_disk("auth/token_store.json") + .build() + .await?; + + let hub = DriveHub::new( + hyper::Client::builder().build( + hyper_rustls::HttpsConnectorBuilder::new() + .with_native_roots() + .https_or_http() + .enable_http1() + .enable_http2() + .build(), + ), + auth, + ); + + let result = hub + .files() + .get("1IotISYu3cF7JrOdfFPKNOkgYg1-ii5Qs") + .doit() + .await?; + // debug!("Result: {:?}", result); + let (body, file) = result; + + debug!("Body: {:?}", body); + debug!("File: {:?}", file); + + // let result = hub.files().list().corpus("user").doit().await; + + // debug!("Result: {:?}", result); + info!("Filename: {:?}", file.name.unwrap_or("NO NAME".to_string())); + info!( + "Description: {:?}", + file.description.unwrap_or("NO DESCRIPTION".to_string()) + ); + Ok(()) +} +use fuser::{ + FileAttr, FileType, Filesystem, MountOption, ReplyAttr, ReplyData, ReplyDirectory, ReplyEmpty, + ReplyEntry, ReplyOpen, ReplyWrite, ReplyXattr, Request, TimeOrNow, FUSE_ROOT_ID, +}; +use std::ffi::OsStr; +use std::path::Path; +use std::time::UNIX_EPOCH; + +#[derive(Default)] +struct MyFS { + /// how long the responses can/should be cached + time_to_live: Duration, + + main_ino: u64, + main_size: u64, + main_blksize: u64, + main_uid: u32, + main_gid: u32, + main_flags: u32, + main_content: Vec, + main_file_type: Option, + main_name: String, +} + +use crate::fs::drive::DriveFilesystem; +use crate::fs::sample::SampleFilesystem; +use async_trait::async_trait; +use tokio::runtime::Runtime; + +struct DirEntry { + ino: u64, + name: String, + file_type: FileType, +} +impl MyFS { + fn get_attr(&self, ino: u64) -> Option { + // Get the file attributes based on the inode number + if ino == FUSE_ROOT_ID { + Some(FileAttr { + ino: FUSE_ROOT_ID, + size: 0, + blocks: 0, + atime: UNIX_EPOCH, + mtime: UNIX_EPOCH, + ctime: UNIX_EPOCH, + crtime: UNIX_EPOCH, + kind: FileType::Directory, + perm: 0o755, + nlink: 0, + uid: 0, + gid: 0, + rdev: 0, + blksize: 0, + flags: 0, + }) + } else if ino == self.main_ino { + Some(FileAttr { + ino: FUSE_ROOT_ID, + size: self.main_size, + blocks: 0, + atime: UNIX_EPOCH, + mtime: UNIX_EPOCH, + ctime: UNIX_EPOCH, + crtime: UNIX_EPOCH, + kind: self.main_file_type.unwrap_or(FileType::RegularFile), + perm: 0o755, + nlink: 0, + uid: self.main_uid, + gid: self.main_gid, + rdev: 0, + blksize: self.main_blksize as u32, + flags: self.main_flags, + }) + } else { + None + } + } + fn set_attr( + &mut self, + ino: u64, + mode: Option, + uid: Option, + gid: Option, + size: Option, + flags: Option, + ) -> Option { + debug!( + "set_attr=> ino: {}; mode: {:?}; uid: {:?}; gid: {:?}; size: {:?}; flags: {:?}", + ino, mode, uid, gid, size, flags + ); + // Get the file attributes based on the inode number + if ino == self.main_ino { + self.main_size = size.unwrap_or(self.main_size); + self.main_flags = flags.unwrap_or(self.main_flags); + self.main_uid = uid.unwrap_or(self.main_uid); + self.main_gid = gid.unwrap_or(self.main_gid); + return self.get_attr(ino); + } else { + None + } + } + fn write_file( + &mut self, + ino: u64, + fh: u64, + offset: i64, + data: &[u8], + flags: i32, + ) -> Option { + // Write the file and reply with the number of bytes written + debug!( + "write_file=> ino: {}; fh: {}; offset: {}; data: {:?}; flags: {}", + ino, fh, offset, data, flags + ); + if ino == self.main_ino { + self.main_content = data.to_vec(); + // todo!("write the file and reply with the number of bytes written"); + return Some(data.len()); + } else { + None + } + } + fn read_file(&self, ino: u64, fh: u64, offset: i64, size: u32) -> Option> { + debug!( + "read_file=> ino: {}; fh: {}; offset: {}; size: {}", + ino, fh, offset, size + ); + if ino == self.main_ino { + // Read the file and reply with the data + let data = &self.main_content.clone(); //b"Hello World!"; + let offset_usize = offset as usize; + let size_usize = size as usize; + if data.len() <= offset_usize { + let result = vec![libc::EOF as u8]; + debug!("read_file=> (0) result: {:?}", result); + return Some(result); + } + if offset_usize + size_usize > data.len() { + //return the rest of the data + EOF + let mut result = data[1..].to_vec(); + result.push(libc::EOF as u8); + debug!("read_file=> (1) result: {:?}", result); + return Some(result); + // todo!("output the rest of the data + EOF, not just EOF"); + return None; + } + + let result = data[offset_usize..offset_usize + size_usize].to_vec(); + debug!("read_file=> (2) result: {:?}", result); + return Some(result); + } else { + None + } + } + fn read_dir(&self, ino: u64) -> Option> { + if ino == FUSE_ROOT_ID { + let mut entries = Vec::new(); + + let dir_entry = DirEntry { + ino: self.main_ino, + name: self.main_name.clone(), + file_type: self.main_file_type.unwrap_or(FileType::RegularFile), + }; + + entries.push(dir_entry); + Some(entries) + } else { + None + } + } +} +#[async_trait] +impl Filesystem for MyFS { + fn open(&mut self, _req: &Request<'_>, _ino: u64, _flags: i32, reply: ReplyOpen) { + if _ino == self.main_ino { + reply.opened(0, 0); + } else { + reply.error(libc::ENOENT); + } + } + fn access(&mut self, _req: &Request<'_>, ino: u64, mask: i32, reply: ReplyEmpty) { + if ino == self.main_ino { + reply.ok() + } else { + reply.error(libc::ENOENT) + } + } + fn getattr(&mut self, _req: &Request, ino: u64, reply: ReplyAttr) { + if let Some(attr) = self.get_attr(ino) { + reply.attr(&self.time_to_live, &attr); + } else { + reply.error(libc::ENOENT); + } + } + fn write( + &mut self, + _req: &Request<'_>, + ino: u64, + fh: u64, + offset: i64, + data: &[u8], + write_flags: u32, + flags: i32, + lock_owner: Option, + reply: ReplyWrite, + ) { + if let Some(size) = self.write_file(ino, fh, offset, data, flags) { + reply.written(size as u32); + } else { + reply.error(libc::ENOENT); + } + } + fn setattr( + &mut self, + _req: &Request<'_>, + ino: u64, + mode: Option, + uid: Option, + gid: Option, + size: Option, + _atime: Option, + _mtime: Option, + _ctime: Option, + fh: Option, + _crtime: Option, + _chgtime: Option, + _bkuptime: Option, + flags: Option, + reply: ReplyAttr, + ) { + if let Some(attr) = self.set_attr(ino, mode, uid, gid, size, flags) { + reply.attr(&self.time_to_live, &attr); + } else { + reply.error(libc::ENOENT); + } + } + fn read( + &mut self, + _req: &Request<'_>, + ino: u64, + fh: u64, + offset: i64, + size: u32, + flags: i32, + lock_owner: Option, + reply: ReplyData, + ) { + if let Some(data) = self.read_file(ino, fh, offset, size) { + let data = data.as_slice(); + reply.data(data); + } else { + reply.error(libc::ENOENT); + } + } + fn readdir( + &mut self, + _req: &Request<'_>, + ino: u64, + fh: u64, + offset: i64, + mut reply: ReplyDirectory, + ) { + debug!("readdir=> ino: {}; fh: {}; offset: {}", ino, fh, offset); + if let Some(entries) = self.read_dir(ino) { + for (i, entry) in entries.iter().enumerate().skip(offset as usize) { + if reply.add(entry.ino, (i + 1) as i64, entry.file_type, &entry.name) { + break; + } + } + reply.ok(); + } else { + reply.error(libc::ENOENT); + } + } + fn lookup(&mut self, _req: &Request<'_>, parent: u64, name: &OsStr, reply: ReplyEntry) { + let main_path = OsStr::new(&self.main_name); + debug!( + "lookup=> parent: {}; name: {:?}; main_path: {:?}", + parent, name, main_path + ); + if name.eq_ignore_ascii_case(main_path) { + let attr = self.get_attr(self.main_ino).unwrap(); + reply.entry(&self.time_to_live, &attr, 0); + } else { + reply.error(libc::ENOENT); + } + } +} +pub async fn watch_file_reading() -> Result<()> { + let mountpoint = "/tmp/fuse/1"; + let options = vec![ + MountOption::RW, + // MountOption::FSName("myfs".to_string()), + // MountOption::AllowOther, + // MountOption::AutoUnmount, + ]; + debug!("Mounting fuse filesystem at {}", mountpoint); + fuser::mount2( + MyFS { + time_to_live: Duration::from_secs(5), + main_ino: 2, + main_name: "1.txt".to_string(), + main_file_type: Some(FileType::RegularFile), + main_content: b"Hello World!".to_vec(), + ..Default::default() + }, + mountpoint, + &options, + ) + .unwrap(); + debug!("Exiting..."); + + Ok(()) +} +pub async fn sample_fs() -> Result<()> { + let mountpoint = "/tmp/fuse/1"; + let source = "/tmp/fuse/2"; + let options = vec![MountOption::RW]; + debug!("Mounting fuse filesystem at {}", mountpoint); + let fs = SampleFilesystem::new(mountpoint, source); + + fuser::mount2(fs, mountpoint, &options).unwrap(); + + debug!("Exiting..."); + Ok(()) +} +pub async fn sample_drive_fs() -> Result<()> { + let mountpoint = "/tmp/fuse/3"; + // let source = "/tmp/fuse/2"; + let options = vec![MountOption::RW]; + debug!("Mounting fuse filesystem at {}", mountpoint); + let fs = DriveFilesystem::new(mountpoint).await?; + + fuser::mount2(fs, mountpoint, &options).unwrap(); + + debug!("Exiting..."); + Ok(()) +} +// pub async fn watch_file_reading() -> Result<()> { +// let temp_file = tempfile::NamedTempFile::new()?; +// let file_path = temp_file.path(); +// info!("File path: {:?}", file_path); +// use notify::{recommended_watcher, RecursiveMode, Watcher}; +// let mut config = notify::Config::default(); +// let mut watcher: INotifyWatcher = Watcher::new(MyReadHandler, config).unwrap(); +// watcher +// .watch(file_path, RecursiveMode::NonRecursive) +// .unwrap(); +// +// info!("Press any key to exit..."); +// let x = &mut [0u8; 1]; +// stdin().read(x).await?; +// debug!("Done"); +// Ok(()) +// } +// struct MyReadHandler; +// impl notify::EventHandler for MyReadHandler { +// fn handle_event(&mut self, event: std::result::Result) { +// debug!("File read: {:?}", event); +// } +// } +// +// pub async fn sample_nix() -> Result<()> { +// info!("Hello, world! (nix)"); +// let tmppath = tempfile::tempdir()?; +// nix::mount::mount( +// // Some("/home/omgeeky/Documents/testmount/"), +// None::<&str>, +// tmppath.path(), +// Some("tmpfs"), +// nix::mount::MsFlags::empty(), +// None::<&str>, +// ); +// info!("Mounted tmpfs at {:?}", tmppath.path()); +// info!("Press any key to exit (nix)..."); +// // block execution until keyboard input is received +// nix::unistd::read(0, &mut [0])?; +// nix::mount::umount(tmppath.path()).unwrap(); +// info!("Done (nix)"); +// Ok(()) +// } diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..5986fbf --- /dev/null +++ b/src/main.rs @@ -0,0 +1,21 @@ +#[tokio::main] +async fn main() { + untitled::init_logger(); + // use tokio::runtime::Runtime; + + // let rt = Runtime::new().unwrap(); + // let filesystem_runtime = Runtime::new().unwrap(); + // + // let handle = rt.handle(); + // handle.block_on(async { + // untitled::sample().await.unwrap(); + // untitled::sample_fs().await.unwrap(); + // untitled::google_drive::sample().await.unwrap(); + // untitled::watch_file_reading().await.unwrap(); + // untitled::sample_nix().await.unwrap(); + untitled::sample_drive_fs().await.unwrap(); + // }); + // RUNTIME.block_on(async { + // //test + // }); +} diff --git a/src/prelude.rs b/src/prelude.rs new file mode 100644 index 0000000..328190c --- /dev/null +++ b/src/prelude.rs @@ -0,0 +1,3 @@ +use std::error::Error; + +pub type Result = core::result::Result>;