Open_Archival/openarchival
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed bug where deletes of artifact groupings would not cascade

Browse Source
This commit is contained in:
Vincent Allen
2025-11-12 19:10:35 -05:00
parent b34449808f
commit 9298829db6
325 changed files with 5233 additions and 20996 deletions
Download Patch File Download Diff File Expand all files Collapse all files

262
OpenArchival.DataAccess/Providers/ArtifactGroupingProvider.cs
View File

@@ -6,7 +6,7 @@ namespace OpenArchival.DataAccess;
public class ArtifactGroupingProvider : IArtifactGroupingProvider
{
private readonly IDbContextFactory<ApplicationDbContext> _context;
private readonly IDbContextFactory<ApplicationDbContext> _context;
private readonly ILogger<ArtifactGroupingProvider> _logger;
[SetsRequiredMembers]
@@ -27,15 +27,14 @@ public class ArtifactGroupingProvider : IArtifactGroupingProvider
.ThenInclude(e => e.StorageLocation)
.Include(g => g.ChildArtifactEntries)
.ThenInclude(e => e.Type)
.Include(g => g.ChildArtifactEntries)
.ThenInclude(e => e.Files)
.Include(g => g.ChildArtifactEntries)
.ThenInclude(e => e.Tags)
.Include(g => g.ChildArtifactEntries)
.ThenInclude(e => e.ListedNames)
.Include(g => g.ChildArtifactEntries)
.ThenInclude(e => e.Defects)
.Include(g => g.ViewCount)
.Include(g => g.ViewCount) // Added
.Include(g => g.IdentifierFields)
.Where(g => g.Id == id)
.FirstOrDefaultAsync();
}
@@ -52,13 +51,12 @@ public class ArtifactGroupingProvider : IArtifactGroupingProvider
.Include(g => g.ChildArtifactEntries)
.ThenInclude(e => e.Type)
.Include(g => g.ChildArtifactEntries)
.ThenInclude(e => e.Files)
.Include(g=> g.ChildArtifactEntries)
.ThenInclude(e => e.Tags)
.Include(g => g.ChildArtifactEntries)
.ThenInclude(e => e.ListedNames)
.Include(g => g.ChildArtifactEntries)
.ThenInclude(e => e.Defects)
.Include(g => g.ViewCount) // Added
.Where(g => g.ArtifactGroupingIdentifier == artifactGroupingIdentifier)
.FirstOrDefaultAsync();
}
@@ -70,149 +68,142 @@ public class ArtifactGroupingProvider : IArtifactGroupingProvider
// Attach the Category to the context. If it has a key, it will be tracked.
context.Attach(grouping.Category);
// --- Local caches for de-duplication within this transaction ---
var processedTypes = new Dictionary<string, ArtifactType>();
var processedLocations = new Dictionary<string, ArtifactStorageLocation>();
var processedTags = new Dictionary<string, ArtifactEntryTag>();
var processedNames = new Dictionary<string, ListedName>();
var processedDefects = new Dictionary<string, ArtifactDefect>();
// --- End local caches ---
// Helper function to get a de-duplicated type
// --- Helper functions to get unique entities (from cache or DB) ---
async Task<ArtifactType> GetUniqueTypeAsync(ArtifactType typeToProcess)
{
// If the type is null or has no name, do nothing.
if (string.IsNullOrEmpty(typeToProcess?.Name))
{
return typeToProcess;
}
// A. First, check our local cache for the type.
if (string.IsNullOrEmpty(typeToProcess?.Name)) return typeToProcess;
if (processedTypes.TryGetValue(typeToProcess.Name, out var uniqueType))
{
// Found it in the cache! Return the single instance we're tracking.
return uniqueType;
}
// B. If not in the cache, check the database.
var dbType = await context.ArtifactTypes.FirstOrDefaultAsync(t => t.Name == typeToProcess.Name);
if (dbType != null)
{
// Found it in the database. Add it to our cache for next time.
processedTypes[dbType.Name] = dbType;
return dbType;
}
// C. It's a brand new type. Add the new instance to our cache.
processedTypes[typeToProcess.Name] = typeToProcess;
return typeToProcess;
}
// 2. De-duplicate the main grouping's type
async Task<ArtifactStorageLocation> GetUniqueLocationAsync(ArtifactStorageLocation locationToProcess)
{
if (string.IsNullOrEmpty(locationToProcess?.Location)) return locationToProcess;
if (processedLocations.TryGetValue(locationToProcess.Location, out var uniqueLocation))
{
return uniqueLocation;
}
var dbLocation = await context.ArtifactStorageLocations.FirstOrDefaultAsync(l => l.Location == locationToProcess.Location);
if (dbLocation != null)
{
processedLocations[dbLocation.Location] = dbLocation;
return dbLocation;
}
processedLocations[locationToProcess.Location] = locationToProcess;
return locationToProcess;
}
async Task<ArtifactEntryTag> GetUniqueTagAsync(ArtifactEntryTag tagToProcess)
{
if (string.IsNullOrEmpty(tagToProcess?.Name)) return tagToProcess;
if (processedTags.TryGetValue(tagToProcess.Name, out var uniqueTag))
{
return uniqueTag;
}
var dbTag = await context.ArtifactEntryTags.FirstOrDefaultAsync(t => t.Name == tagToProcess.Name);
if (dbTag != null)
{
processedTags[dbTag.Name] = dbTag;
return dbTag;
}
processedTags[tagToProcess.Name] = tagToProcess;
return tagToProcess;
}
async Task<ListedName> GetUniqueNameAsync(ListedName nameToProcess)
{
if (string.IsNullOrEmpty(nameToProcess?.Value)) return nameToProcess;
if (processedNames.TryGetValue(nameToProcess.Value, out var uniqueName))
{
return uniqueName;
}
var dbName = await context.ArtifactAssociatedNames.FirstOrDefaultAsync(n => n.Value == nameToProcess.Value);
if (dbName != null)
{
processedNames[dbName.Value] = dbName;
return dbName;
}
processedNames[nameToProcess.Value] = nameToProcess;
return nameToProcess;
}
async Task<ArtifactDefect> GetUniqueDefectAsync(ArtifactDefect defectToProcess)
{
if (string.IsNullOrEmpty(defectToProcess?.Description)) return defectToProcess;
if (processedDefects.TryGetValue(defectToProcess.Description, out var uniqueDefect))
{
return uniqueDefect;
}
var dbDefect = await context.ArtifactDefects.FirstOrDefaultAsync(d => d.Description == defectToProcess.Description);
if (dbDefect != null)
{
processedDefects[dbDefect.Description] = dbDefect;
return dbDefect;
}
processedDefects[defectToProcess.Description] = defectToProcess;
return defectToProcess;
}
// --- End helper functions ---
// De-duplicate the main grouping's type
grouping.Type = await GetUniqueTypeAsync(grouping.Type);
// Iterate through all child entries to handle their related entities.
foreach (var entry in grouping.ChildArtifactEntries)
{
// Handle Artifact Types
// Check if the type exists in the database.
var existingType = await GetUniqueTypeAsync(entry.Type);
entry.Type = existingType;
entry.Type = await GetUniqueTypeAsync(entry.Type);
// Handle Storage Location
// Check if the storage location exists in the database.
var existingLocation = await context.ArtifactStorageLocations.FirstOrDefaultAsync(l => l.Location == entry.StorageLocation.Location);
if (existingLocation != null)
{
// If it exists, replace the disconnected object with the tracked one.
entry.StorageLocation = existingLocation;
}
entry.StorageLocation = await GetUniqueLocationAsync(entry.StorageLocation);
// Handle Tags
// Create a temporary list to hold the managed tag entities.
var managedTags = new List<ArtifactEntryTag>();
foreach (var tag in entry.Tags)
{
// Attempt to find the tag in the database.
var existingTag = await context.ArtifactEntryTags.FirstOrDefaultAsync(t => t.Name == tag.Name);
if (existingTag != null)
{
// The tag already exists. Use the tracked instance.
managedTags.Add(existingTag);
}
else
{
// The tag is new. Add it to the managed list.
managedTags.Add(tag);
}
managedTags.Add(await GetUniqueTagAsync(tag));
}
// Replace the disconnected tag objects on the entry with the managed ones.
entry.Tags = managedTags;
// Handle Listed Names
// Create a temporary list to hold the managed name entities.
var managedNames = new List<ListedName>();
foreach (var name in entry.ListedNames)
{
// Attempt to find the listed name in the database.
var existingName = await context.ArtifactAssociatedNames.FirstOrDefaultAsync(n => n.Value == name.Value);
if (existingName != null)
{
// The name already exists. Use the tracked instance.
managedNames.Add(existingName);
}
else
{
// The name is new. Add it to the managed list.
managedNames.Add(name);
}
managedNames.Add(await GetUniqueNameAsync(name));
}
// Replace the disconnected name objects on the entry with the managed ones.
entry.ListedNames = managedNames;
// Handle Defects
// Create a temporary list to hold the managed defect entities.
var managedDefects = new List<ArtifactDefect>();
foreach (var defect in entry.Defects)
{
// Attempt to find the defect in the database.
var existingDefect = await context.ArtifactDefects.FirstOrDefaultAsync(d => d.Description == defect.Description);
if (existingDefect != null)
{
// The defect already exists. Use the tracked instance.
managedDefects.Add(existingDefect);
}
else
{
// The defect is new. Add it to the managed list.
managedDefects.Add(defect);
}
managedDefects.Add(await GetUniqueDefectAsync(defect));
}
// Replace the disconnected defect objects on the entry with the managed ones.
entry.Defects = managedDefects;
// Handle file paths. This is the original logic you provided.
var managedFilePaths = new List<FilePathListing>();
foreach (var filepath in entry.Files)
{
// Attempt to find the file path in the database.
var existingFilePath = await context.ArtifactFilePaths.FirstOrDefaultAsync(f => f.Path == filepath.Path);
if (existingFilePath != null)
{
// The file path already exists. Use the tracked instance.
managedFilePaths.Add(existingFilePath);
}
else
{
// The file path is new. Add it to the managed list.
managedFilePaths.Add(filepath);
}
}
// Replace the disconnected file path objects on the entry with the managed ones.
entry.Files = managedFilePaths;
}
// Concatinate all of the text to be searchable by postgres
grouping.GenerateSearchIndex();
// Add the new grouping and save changes.
//context.ArtifactGroupings.Add(grouping);
context.ChangeTracker.TrackGraph(grouping, node =>
{
// If the entity's key is set, EF should treat it as an existing, unchanged entity.
@@ -232,22 +223,19 @@ public class ArtifactGroupingProvider : IArtifactGroupingProvider
public async Task UpdateGroupingAsync(ArtifactGrouping updatedGrouping)
{
// The DbContext is provided externally, so we will use it as is.
// Assuming you have an instance available, e.g., via a constructor or method parameter.
await using var context = await _context.CreateDbContextAsync();
// 1. Retrieve the existing grouping object from the database, eagerly loading all related data.
// This is crucial for correctly handling all relationships.
var existingGrouping = await context.ArtifactGroupings
.Include(g => g.Category)
.Include(g => g.IdentifierFields)
.Include(g => g.Type)
.Include(g => g.ViewCount) // Load ViewCount
//.Include(g => g.BlogPosts) // BlogPosts not handled yet
.Include(g => g.ChildArtifactEntries)
.ThenInclude(e => e.StorageLocation)
.Include(g => g.ChildArtifactEntries)
.ThenInclude(e => e.Type)
.Include(g => g.ChildArtifactEntries)
.ThenInclude(e => e.Files)
.Include(g => g.ChildArtifactEntries)
.ThenInclude(e => e.Tags)
.Include(g => g.ChildArtifactEntries)
@@ -269,7 +257,6 @@ public class ArtifactGroupingProvider : IArtifactGroupingProvider
existingGrouping.IdentifierFields = updatedGrouping.IdentifierFields;
// Handle one-to-many relationships (Type, Category).
// Find the existing related entity and attach it to the tracked graph.
var existingGroupingType = await context.ArtifactTypes.FirstOrDefaultAsync(t => t.Name == updatedGrouping.Type.Name);
if (existingGroupingType != null)
{
@@ -280,20 +267,37 @@ public class ArtifactGroupingProvider : IArtifactGroupingProvider
existingGrouping.Type = updatedGrouping.Type;
}
// Attach the category as specified
if (existingGrouping.Category.Name != updatedGrouping.Category.Name)
{
existingGrouping.Category = updatedGrouping.Category;
context.Add(existingGrouping.Category);
}
// Update top-level properties.
existingGrouping.Title = updatedGrouping.Title;
existingGrouping.IsPublicallyVisible = updatedGrouping.IsPublicallyVisible;
existingGrouping.Description = updatedGrouping.Description;
// Handle ViewCount (Added)
if (updatedGrouping.ViewCount != null)
{
if (existingGrouping.ViewCount == null)
{
// Create a new ViewCount
existingGrouping.ViewCount = new ArtifactGroupingViewCount
{
Grouping = existingGrouping,
Views = updatedGrouping.ViewCount.Views
};
}
else
{
// Update existing ViewCount
existingGrouping.ViewCount.Views = updatedGrouping.ViewCount.Views;
}
}
// TODO: Handle BlogPosts update (requires model definition & de-duplication)
// await DeDuplicateGroupingRelationsAsync(context, updatedGrouping);
// existingGrouping.BlogPosts.Clear();
// updatedGrouping.BlogPosts.ForEach(post => existingGrouping.BlogPosts.Add(post));
// 3. Synchronize the ChildArtifactEntries collection.
// First, remove any entries that were deleted in the DTO.
var updatedEntryIds = updatedGrouping.ChildArtifactEntries.Select(e => e.Id).ToList();
var entriesToRemove = existingGrouping.ChildArtifactEntries
.Where(e => !updatedEntryIds.Contains(e.Id))
@@ -304,15 +308,13 @@ public class ArtifactGroupingProvider : IArtifactGroupingProvider
existingGrouping.ChildArtifactEntries.Remove(entryToRemove);
}
// Now, loop through the updated entries to handle updates and additions.
foreach (var updatedEntry in updatedGrouping.ChildArtifactEntries)
{
// FIRST, de-duplicate all related entities on the incoming entry.
await DeDuplicateEntryRelationsAsync(context, updatedEntry);
var existingEntry = existingGrouping.ChildArtifactEntries
.FirstOrDefault(e => e.Id == updatedEntry.Id);
await DeDuplicateEntryRelationsAsync(context, updatedEntry);
if (existingEntry != null)
{
@@ -323,8 +325,8 @@ public class ArtifactGroupingProvider : IArtifactGroupingProvider
existingEntry.IsPubliclyVisible = updatedEntry.IsPubliclyVisible;
existingEntry.AssociatedDates = updatedEntry.AssociatedDates;
existingEntry.FileTextContent = updatedEntry.FileTextContent;
existingEntry.Files = updatedEntry.Files;
existingEntry.Quantity = updatedEntry.Quantity;
existingEntry.Links = updatedEntry.Links;
// The relations on updatedEntry are already de-duplicated, so just assign them.
existingEntry.StorageLocation = updatedEntry.StorageLocation;
@@ -382,14 +384,14 @@ public class ArtifactGroupingProvider : IArtifactGroupingProvider
// De-duplicate ListedNames
var processedNames = new List<ListedName>();
if (entry.ListedNames != null)
{
{
foreach (var name in entry.ListedNames)
{
{
var existingName = await context.ArtifactAssociatedNames.FirstOrDefaultAsync(n => n.Value == name.Value) ?? name;
processedNames.Add(existingName);
}
entry.ListedNames = processedNames;
}
}
// De-duplicate Defects
var processedDefects = new List<ArtifactDefect>();
@@ -399,22 +401,8 @@ public class ArtifactGroupingProvider : IArtifactGroupingProvider
{
var existingDefect = await context.ArtifactDefects.FirstOrDefaultAsync(d => d.Description == defect.Description) ?? defect;
processedDefects.Add(existingDefect);
}
entry.Defects = processedDefects;
}
if (entry.Files.Any())
{
// 1. Get the IDs from the incoming, untracked file objects.
var inputFileIds = entry.Files.Select(f => f.Id).ToList();
// 2. Fetch the actual, tracked entities from the database.
var trackedFiles = await context.ArtifactFilePaths
.Where(dbFile => inputFileIds.Contains(dbFile.Id))
.ToListAsync();
// 3. Replace the untracked collection with the tracked one.
entry.Files = trackedFiles;
entry.Defects = processedDefects;
}
}
@@ -441,12 +429,11 @@ public class ArtifactGroupingProvider : IArtifactGroupingProvider
// 2. Identify keys for brand new items
var keysToAdd = updatedKeys.Except(existingKeys).ToList();
if (!keysToAdd.Any())
{
{
return; // Nothing to add
}
}
// 3. Batch-fetch all entities from the DB that match the new keys.
// This is the key change to make the query translatable to SQL.
Dictionary<TKey, TEntity> existingDbItemsMap = [];
if (typeof(TEntity) == typeof(ArtifactEntryTag))
{
@@ -463,7 +450,7 @@ public class ArtifactGroupingProvider : IArtifactGroupingProvider
.Where(n => nameKeys.Contains(n.Value))
.ToListAsync();
existingDbItemsMap = names.ToDictionary(n => (TKey)(object)n.Value) as Dictionary<TKey, TEntity>;
}
}
else if (typeof(TEntity) == typeof(ArtifactDefect))
{
var defectKeys = keysToAdd.Cast<string>().ToList();
@@ -472,6 +459,7 @@ public class ArtifactGroupingProvider : IArtifactGroupingProvider
.ToListAsync();
existingDbItemsMap = defects.ToDictionary(d => (TKey)(object)d.Description) as Dictionary<TKey, TEntity>;
}
// TODO: Add support for other entity types like BlogPost or ArtifactEntry if needed
// 4. Add the items, using the tracked entity from the DB if it exists.
@@ -507,7 +495,7 @@ public class ArtifactGroupingProvider : IArtifactGroupingProvider
context.ArtifactGroupings.Remove(grouping);
await context.SaveChangesAsync();
}
public async Task<List<ArtifactGrouping>> GetGroupingsPaged(int pageNumber, int resultsCount)
{
await using var context = await _context.CreateDbContextAsync();
@@ -525,7 +513,7 @@ public class ArtifactGroupingProvider : IArtifactGroupingProvider
.Skip((pageNumber - 1) * resultsCount)
.Take(resultsCount)
.ToListAsync();
return items;
}