From 3181e4e96e9e952b92b79ed07653fe6b6a46c322 Mon Sep 17 00:00:00 2001 From: BlackDex Date: Wed, 11 Jan 2023 20:23:53 +0100 Subject: [PATCH] Optimize CipherSyncData for very large vaults As mentioned in #3111, using a very very large vault causes some issues. Mainly because of a SQLite limit, but, it could also cause issue on MariaDB/MySQL or PostgreSQL. It also uses a lot of memory, and memory allocations. This PR solves this by removing the need of all the cipher_uuid's just to gather the correct attachments. It will use the user_uuid and org_uuid's to get all attachments linked to both, weither the user has access to them or not. This isn't an issue, since the matching is done per cipher and the attachment data is only returned if there is a matching cipher to where the user has access to. I also modified some code to be able to use `::with_capacity(n)` where possible. This prevents re-allocations if the `Vec` increases size, which will happen a lot if there are a lot of ciphers. According to my tests measuring the time it takes to sync, it seems to have lowered the duration a bit more. Fixes #3111 --- src/api/core/ciphers.rs | 43 ++++++++++++++++++-------------- src/api/core/emergency_access.rs | 5 ++-- src/api/core/organizations.rs | 4 +-- src/db/models/attachment.rs | 9 +++++-- 4 files changed, 35 insertions(+), 26 deletions(-) diff --git a/src/api/core/ciphers.rs b/src/api/core/ciphers.rs index f97403a0..0b0f1080 100644 --- a/src/api/core/ciphers.rs +++ b/src/api/core/ciphers.rs @@ -104,16 +104,17 @@ async fn sync(data: SyncData, headers: Headers, mut conn: DbConn) -> Json // Get all ciphers which are visible by the user let ciphers = Cipher::find_by_user_visible(&headers.user.uuid, &mut conn).await; - let cipher_sync_data = CipherSyncData::new(&headers.user.uuid, &ciphers, CipherSyncType::User, &mut conn).await; + let cipher_sync_data = CipherSyncData::new(&headers.user.uuid, CipherSyncType::User, &mut conn).await; // Lets generate the ciphers_json using all the gathered info - let mut ciphers_json = Vec::new(); + let mut ciphers_json = Vec::with_capacity(ciphers.len()); for c in ciphers { ciphers_json.push(c.to_json(&headers.host, &headers.user.uuid, Some(&cipher_sync_data), &mut conn).await); } - let mut collections_json = Vec::new(); - for c in Collection::find_by_user_uuid(headers.user.uuid.clone(), &mut conn).await { + let collections = Collection::find_by_user_uuid(headers.user.uuid.clone(), &mut conn).await; + let mut collections_json = Vec::with_capacity(collections.len()); + for c in collections { collections_json.push(c.to_json_details(&headers.user.uuid, Some(&cipher_sync_data), &mut conn).await); } @@ -148,9 +149,9 @@ async fn sync(data: SyncData, headers: Headers, mut conn: DbConn) -> Json #[get("/ciphers")] async fn get_ciphers(headers: Headers, mut conn: DbConn) -> Json { let ciphers = Cipher::find_by_user_visible(&headers.user.uuid, &mut conn).await; - let cipher_sync_data = CipherSyncData::new(&headers.user.uuid, &ciphers, CipherSyncType::User, &mut conn).await; + let cipher_sync_data = CipherSyncData::new(&headers.user.uuid, CipherSyncType::User, &mut conn).await; - let mut ciphers_json = Vec::new(); + let mut ciphers_json = Vec::with_capacity(ciphers.len()); for c in ciphers { ciphers_json.push(c.to_json(&headers.host, &headers.user.uuid, Some(&cipher_sync_data), &mut conn).await); } @@ -1721,12 +1722,9 @@ pub enum CipherSyncType { } impl CipherSyncData { - pub async fn new(user_uuid: &str, ciphers: &[Cipher], sync_type: CipherSyncType, conn: &mut DbConn) -> Self { - // Generate a list of Cipher UUID's to be used during a query filter with an eq_any. - let cipher_uuids = ciphers.iter().map(|c| c.uuid.clone()).collect(); - - let mut cipher_folders: HashMap = HashMap::new(); - let mut cipher_favorites: HashSet = HashSet::new(); + pub async fn new(user_uuid: &str, sync_type: CipherSyncType, conn: &mut DbConn) -> Self { + let cipher_folders: HashMap; + let cipher_favorites: HashSet; match sync_type { // User Sync supports Folders and Favorits CipherSyncType::User => { @@ -1738,18 +1736,25 @@ impl CipherSyncData { } // Organization Sync does not support Folders and Favorits. // If these are set, it will cause issues in the web-vault. - CipherSyncType::Organization => {} + CipherSyncType::Organization => { + cipher_folders = HashMap::with_capacity(0); + cipher_favorites = HashSet::with_capacity(0); + } } // Generate a list of Cipher UUID's containing a Vec with one or more Attachment records - let mut cipher_attachments: HashMap> = HashMap::new(); - for attachment in Attachment::find_all_by_ciphers(&cipher_uuids, conn).await { + let user_org_uuids = UserOrganization::get_org_uuid_by_user(user_uuid, conn).await; + let attachments = Attachment::find_all_by_user_and_orgs(user_uuid, &user_org_uuids, conn).await; + let mut cipher_attachments: HashMap> = HashMap::with_capacity(attachments.len()); + for attachment in attachments { cipher_attachments.entry(attachment.cipher_uuid.clone()).or_default().push(attachment); } // Generate a HashMap with the Cipher UUID as key and one or more Collection UUID's - let mut cipher_collections: HashMap> = HashMap::new(); - for (cipher, collection) in Cipher::get_collections_with_cipher_by_user(user_uuid.to_string(), conn).await { + let user_cipher_collections = Cipher::get_collections_with_cipher_by_user(user_uuid.to_string(), conn).await; + let mut cipher_collections: HashMap> = + HashMap::with_capacity(user_cipher_collections.len()); + for (cipher, collection) in user_cipher_collections { cipher_collections.entry(cipher).or_default().push(collection); } @@ -1768,14 +1773,14 @@ impl CipherSyncData { .collect(); // Generate a HashMap with the collections_uuid as key and the CollectionGroup record - let user_collections_groups = CollectionGroup::find_by_user(user_uuid, conn) + let user_collections_groups: HashMap = CollectionGroup::find_by_user(user_uuid, conn) .await .into_iter() .map(|collection_group| (collection_group.collections_uuid.clone(), collection_group)) .collect(); // Get all organizations that the user has full access to via group assignement - let user_group_full_access_for_organizations = + let user_group_full_access_for_organizations: HashSet = Group::gather_user_organizations_full_access(user_uuid, conn).await.into_iter().collect(); Self { diff --git a/src/api/core/emergency_access.rs b/src/api/core/emergency_access.rs index 7a683ea4..f15c1b8e 100644 --- a/src/api/core/emergency_access.rs +++ b/src/api/core/emergency_access.rs @@ -584,10 +584,9 @@ async fn view_emergency_access(emer_id: String, headers: Headers, mut conn: DbCo } let ciphers = Cipher::find_owned_by_user(&emergency_access.grantor_uuid, &mut conn).await; - let cipher_sync_data = - CipherSyncData::new(&emergency_access.grantor_uuid, &ciphers, CipherSyncType::User, &mut conn).await; + let cipher_sync_data = CipherSyncData::new(&emergency_access.grantor_uuid, CipherSyncType::User, &mut conn).await; - let mut ciphers_json = Vec::new(); + let mut ciphers_json = Vec::with_capacity(ciphers.len()); for c in ciphers { ciphers_json .push(c.to_json(&headers.host, &emergency_access.grantor_uuid, Some(&cipher_sync_data), &mut conn).await); diff --git a/src/api/core/organizations.rs b/src/api/core/organizations.rs index 3fb83ae2..c0af8f6e 100644 --- a/src/api/core/organizations.rs +++ b/src/api/core/organizations.rs @@ -617,9 +617,9 @@ async fn get_org_details(data: OrgIdData, headers: Headers, mut conn: DbConn) -> async fn _get_org_details(org_id: &str, host: &str, user_uuid: &str, conn: &mut DbConn) -> Value { let ciphers = Cipher::find_by_org(org_id, conn).await; - let cipher_sync_data = CipherSyncData::new(user_uuid, &ciphers, CipherSyncType::Organization, conn).await; + let cipher_sync_data = CipherSyncData::new(user_uuid, CipherSyncType::Organization, conn).await; - let mut ciphers_json = Vec::new(); + let mut ciphers_json = Vec::with_capacity(ciphers.len()); for c in ciphers { ciphers_json.push(c.to_json(host, user_uuid, Some(&cipher_sync_data), conn).await); } diff --git a/src/db/models/attachment.rs b/src/db/models/attachment.rs index 325b9a27..e850537d 100644 --- a/src/db/models/attachment.rs +++ b/src/db/models/attachment.rs @@ -187,10 +187,15 @@ impl Attachment { }} } - pub async fn find_all_by_ciphers(cipher_uuids: &Vec, conn: &mut DbConn) -> Vec { + // This will return all attachments linked to the user or org + // There is no filtering done here if the user actually has access! + // It is used to speed up the sync process, and the matching is done in a different part. + pub async fn find_all_by_user_and_orgs(user_uuid: &str, org_uuids: &Vec, conn: &mut DbConn) -> Vec { db_run! { conn: { attachments::table - .filter(attachments::cipher_uuid.eq_any(cipher_uuids)) + .left_join(ciphers::table.on(ciphers::uuid.eq(attachments::cipher_uuid))) + .filter(ciphers::user_uuid.eq(user_uuid)) + .or_filter(ciphers::organization_uuid.eq_any(org_uuids)) .select(attachments::all_columns) .load::(conn) .expect("Error loading attachments")