Skip to content

Commit 0c7bf2d

Browse files
habarthierry-huehabarthierry-hue
committed
Fix performance issue with MongoDB
1 parent 504761b commit 0c7bf2d

3 files changed

Lines changed: 181 additions & 73 deletions

File tree

‎src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/Extensions/MongoDbClientExtensions.cs‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -52,15 +52,21 @@ public static void EnsureMongoDbSCIMDatabaseIsCreated(MongoDbOptions options, Li
5252
}
5353
}
5454

55-
private static async void EnsureSCIMRepresentationAttributeIndexesAreCreated(IMongoDatabase db, string name)
56-
{
57-
const string indexName = "RepresentationId_1_SchemaAttributeId_1_ValueString_1";
58-
var collection = db.GetCollection<SCIMRepresentationAttribute>(name);
55+
private static void EnsureSCIMRepresentationAttributeIndexesAreCreated(IMongoDatabase db, string name)
56+
{
57+
var compoundIndex = Builders<SCIMRepresentationAttribute>.IndexKeys.Ascending(a => a.RepresentationId).Ascending(a => a.SchemaAttributeId).Ascending(a => a.ValueString);
58+
var representationIdIndex = Builders<SCIMRepresentationAttribute>.IndexKeys.Ascending(a => a.RepresentationId);
59+
EnsureIndexCreated(db, "RepresentationId_1_SchemaAttributeId_1_ValueString_1", name, compoundIndex);
60+
EnsureIndexCreated(db, "RepresentationId_1", name, representationIdIndex);
61+
}
62+
63+
private static async void EnsureIndexCreated(IMongoDatabase db, string indexName, string name, IndexKeysDefinition<SCIMRepresentationAttribute> indexDefinition)
64+
{
65+
var collection = db.GetCollection<SCIMRepresentationAttribute>(name);
5966
var indexes = await collection.Indexes.List().ToListAsync();
60-
if (indexes.Any(i => i.Elements.Any(e => e.Name == "name" && e.Value.AsString == "RepresentationId_1_SchemaAttributeId_1_ValueString_1"))) return;
61-
var indexDefinition = Builders<SCIMRepresentationAttribute>.IndexKeys.Ascending(a => a.RepresentationId).Ascending(a => a.SchemaAttributeId).Ascending(a => a.ValueString);
62-
collection.Indexes.CreateOne(indexDefinition);
63-
}
67+
if (indexes.Any(i => i.Elements.Any(e => e.Name == "name" && e.Value.AsString == indexName))) return;
68+
collection.Indexes.CreateOne(indexDefinition);
69+
}
6470

6571
private static IMongoCollection<T> EnsureCollectionIsCreated<T>(IMongoDatabase db, string name)
6672
{

‎src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/Extensions/MongoDbSCIMExpressionLinqExtensions.cs‎

Lines changed: 3 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,8 @@
11
// Copyright (c) SimpleIdServer. All rights reserved.
22
// Licensed under the Apache License, Version 2.0. See LICENSE in the project root for license information.
33
using MongoDB.Driver.Linq;
4-
using SimpleIdServer.Persistence.Filters;
5-
using SimpleIdServer.Scim.Domains;
64
using SimpleIdServer.Scim.Parser.Expressions;
75
using SimpleIdServer.Scim.Parser.Operators;
8-
using SimpleIdServer.Scim.Persistence.MongoDB.Models;
96
using System;
107
using System.Linq;
118
using System.Linq.Expressions;
@@ -77,55 +74,16 @@ public static Expression EvaluateAttributes(this SCIMComparisonExpression expres
7774
var propertyValueBoolean = Expression.Property(attr, "ValueBoolean");
7875
var propertyValueDecimal = Expression.Property(attr, "ValueDecimal");
7976
var propertyValueBinary = Expression.Property(attr, "ValueBinary");
80-
var comparison = SCIMExpressionLinqExtensions.BuildComparisonExpression(expression, expression.LeftExpression.SchemaAttribute,
77+
var lastChild = expression.LeftExpression.GetLastChild();
78+
var comparison = SCIMExpressionLinqExtensions.BuildComparisonExpression(expression, lastChild.SchemaAttribute,
8179
propertyValueString,
8280
propertyValueInteger,
8381
propertyValueDatetime,
8482
propertyValueBoolean,
8583
propertyValueDecimal,
8684
propertyValueBinary,
8785
parameterExpression);
88-
return Expression.And(Expression.Equal(schemaAttributeId, Expression.Constant(expression.LeftExpression.SchemaAttribute.Id)), comparison);
89-
}
90-
91-
#endregion
92-
93-
#region Order By
94-
95-
public static IOrderedMongoQueryable<SCIMRepresentationModel> EvaluateMongoDbOrderBy(this SCIMExpression expression, IMongoQueryable<SCIMRepresentationModel> representations, SearchSCIMRepresentationOrders order)
96-
{
97-
var attrExpression = expression as SCIMAttributeExpression;
98-
if (attrExpression == null)
99-
{
100-
return null;
101-
}
102-
103-
var result = EvaluateOrderByMetadata(attrExpression, representations, order);
104-
return result;
105-
}
106-
107-
public static IOrderedMongoQueryable<SCIMRepresentationModel> EvaluateOrderByMetadata(SCIMAttributeExpression attrExpression, IMongoQueryable<SCIMRepresentationModel> representations, SearchSCIMRepresentationOrders order)
108-
{
109-
var fullPath = attrExpression.GetFullPath();
110-
switch(fullPath)
111-
{
112-
case StandardSCIMRepresentationAttributes.Id:
113-
return order == SearchSCIMRepresentationOrders.Ascending ? representations.OrderBy(r => r.Id) : representations.OrderByDescending(r => r.Id);
114-
case StandardSCIMRepresentationAttributes.ExternalId:
115-
return order == SearchSCIMRepresentationOrders.Ascending ? representations.OrderBy(r => r.ExternalId) : representations.OrderByDescending(r => r.ExternalId);
116-
case $"{StandardSCIMRepresentationAttributes.Meta}.{StandardSCIMMetaAttributes.ResourceType}":
117-
return order == SearchSCIMRepresentationOrders.Ascending ? representations.OrderBy(r => r.ResourceType) : representations.OrderByDescending(r => r.ResourceType);
118-
case $"{StandardSCIMRepresentationAttributes.Meta}.{StandardSCIMMetaAttributes.Created}":
119-
return order == SearchSCIMRepresentationOrders.Ascending ? representations.OrderBy(r => r.Created) : representations.OrderByDescending(r => r.Created);
120-
case $"{StandardSCIMRepresentationAttributes.Meta}.{StandardSCIMMetaAttributes.LastModified}":
121-
return order == SearchSCIMRepresentationOrders.Ascending ? representations.OrderBy(r => r.LastModified) : representations.OrderByDescending(r => r.LastModified);
122-
case $"{StandardSCIMRepresentationAttributes.Meta}.{StandardSCIMMetaAttributes.Version}":
123-
return order == SearchSCIMRepresentationOrders.Ascending ? representations.OrderBy(r => r.Version) : representations.OrderByDescending(r => r.Version);
124-
case StandardSCIMRepresentationAttributes.DisplayName:
125-
return order == SearchSCIMRepresentationOrders.Ascending ? representations.OrderBy(r => r.DisplayName) : representations.OrderByDescending(r => r.DisplayName);
126-
}
127-
128-
return representations.OrderBy(r => r.Id);
86+
return Expression.And(Expression.Equal(schemaAttributeId, Expression.Constant(lastChild.SchemaAttribute.Id)), comparison);
12987
}
13088

13189
#endregion

‎src/Scim/SimpleIdServer.Scim.Persistence.MongoDB/SCIMRepresentationQueryRepository.cs‎

Lines changed: 164 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,9 @@
66
using SimpleIdServer.Persistence.Filters;
77
using SimpleIdServer.Scim.Domain;
88
using SimpleIdServer.Scim.Domains;
9+
using SimpleIdServer.Scim.Parser.Expressions;
910
using SimpleIdServer.Scim.Persistence.MongoDB.Extensions;
10-
using System;
11+
using SimpleIdServer.Scim.Persistence.MongoDB.Models;
1112
using System.Collections.Generic;
1213
using System.Linq;
1314
using System.Threading;
@@ -63,7 +64,7 @@ public async Task<SearchSCIMRepresentationsResponse> FindSCIMRepresentations(Sea
6364
int total = 0;
6465
if (parameter.Filter != null)
6566
{
66-
IQueryable<EnrichedAttribute> filteredRepresentationAttributes = from a in _scimDbContext.SCIMRepresentationAttributeLst.AsQueryable().Where(a => parameter.SchemaNames.Contains(a.Namespace))
67+
var filteredRepresentationAttributes = from a in _scimDbContext.SCIMRepresentationAttributeLst.AsQueryable().Where(a => parameter.SchemaNames.Contains(a.Namespace) || a.ResourceType == parameter.ResourceType)
6768
join b in _scimDbContext.SCIMRepresentationAttributeLst.AsQueryable() on a.ParentAttributeId equals b.Id into Parents
6869
select new EnrichedAttribute
6970
{
@@ -86,32 +87,175 @@ join b in _scimDbContext.SCIMRepresentationAttributeLst.AsQueryable() on a.Paren
8687
.CountAsync();
8788
}
8889

89-
90-
var filteredRepresentations = _scimDbContext.SCIMRepresentationLst.AsQueryable().Where(s => s.ResourceType == parameter.ResourceType);
90+
91+
var filteredRepresentations = _scimDbContext.SCIMRepresentationLst.AsQueryable()
92+
.Where(r => r.ResourceType == parameter.ResourceType);
9193
if(filteredRepresentationIds != null)
94+
{
9295
filteredRepresentations = filteredRepresentations.Where(r => filteredRepresentationIds.Contains(r.Id));
96+
}
9397

94-
if (parameter.SortBy == null)
95-
filteredRepresentations = filteredRepresentations.OrderBy(s => s.Id);
96-
else
98+
var paginationResult = await OrderByAndPaginate(filteredRepresentations, parameter);
99+
filteredRepresentations = paginationResult.Query;
100+
var filteredRepresentationsWithAttributes = from a in filteredRepresentations
101+
join b in _scimDbContext.SCIMRepresentationAttributeLst.AsQueryable() on a.Id equals b.RepresentationId into Attributes
102+
select new
103+
{
104+
Representation = a,
105+
Attributes = Attributes
106+
};
107+
var representationWithAttributes = await filteredRepresentationsWithAttributes
108+
.ToListAsync();
109+
if(paginationResult.OrderedRepresentationIds != null)
97110
{
98-
filteredRepresentations = parameter.SortBy.EvaluateMongoDbOrderBy(
99-
filteredRepresentations,
100-
parameter.SortOrder ?? SearchSCIMRepresentationOrders.Descending);
111+
representationWithAttributes = representationWithAttributes.Select(r => new
112+
{
113+
Representation = r,
114+
Order = paginationResult.OrderedRepresentationIds.IndexOf(r.Representation.Id)
115+
})
116+
.OrderBy(r => r.Order)
117+
.Select(r => r.Representation)
118+
.ToList();
119+
}
120+
121+
var representations = new List<SCIMRepresentation>();
122+
foreach(var record in representationWithAttributes)
123+
{
124+
var representation = record.Representation;
125+
representation.FlatAttributes = record.Attributes.ToList();
126+
representations.Add(representation);
101127
}
102128

103-
var representations = await filteredRepresentations
104-
.Skip(parameter.StartIndex <= 1 ? 0 : parameter.StartIndex - 1)
105-
.Take(parameter.Count)
106-
.ToListAsync();
107-
var representationIds = representations.Select(r => r.Id);
108-
var representationAttributes = await _scimDbContext.SCIMRepresentationAttributeLst.AsQueryable()
109-
.Where(a => representationIds.Contains(a.RepresentationId))
110-
.ToListAsync();
111-
foreach (var representation in representations)
112-
representation.FlatAttributes = representationAttributes.Where(a => a.RepresentationId == representation.Id).ToList();
113129
representations.FilterAttributes(parameter.IncludedAttributes, parameter.ExcludedAttributes);
114130
return new SearchSCIMRepresentationsResponse(total, representations);
115131
}
132+
133+
private async Task<PaginationResult> OrderByAndPaginate(
134+
IMongoQueryable<SCIMRepresentationModel> representations,
135+
SearchSCIMRepresentationsParameter parameter)
136+
{
137+
if (parameter.SortBy == null) return new PaginationResult
138+
{
139+
Query = representations
140+
.OrderBy(r => r.Id)
141+
.Skip(parameter.StartIndex <= 1 ? 0 : parameter.StartIndex - 1)
142+
.Take(parameter.Count)
143+
};
144+
var sortBy = parameter.SortBy as SCIMAttributeExpression;
145+
var order = parameter.SortOrder ?? SearchSCIMRepresentationOrders.Ascending;
146+
var result = OrderByMetadataAndPaginate(sortBy,
147+
representations,
148+
order,
149+
parameter.StartIndex,
150+
parameter.Count);
151+
if (result != null) return result;
152+
return await OrderByPropertyAndPaginate(
153+
representations,
154+
sortBy,
155+
parameter.SchemaNames,
156+
order,
157+
parameter.StartIndex,
158+
parameter.Count);
159+
}
160+
161+
private PaginationResult OrderByMetadataAndPaginate(
162+
SCIMAttributeExpression attrExpression,
163+
IMongoQueryable<SCIMRepresentationModel> representations,
164+
SearchSCIMRepresentationOrders order,
165+
int startIndex,
166+
int count)
167+
{
168+
var fullPath = attrExpression.GetFullPath();
169+
IMongoQueryable<SCIMRepresentationModel> result = null;
170+
switch (fullPath)
171+
{
172+
case StandardSCIMRepresentationAttributes.Id:
173+
result = order == SearchSCIMRepresentationOrders.Ascending ? representations.OrderBy(r => r.Id) : representations.OrderByDescending(r => r.Id);
174+
break;
175+
case StandardSCIMRepresentationAttributes.ExternalId:
176+
result = order == SearchSCIMRepresentationOrders.Ascending ? representations.OrderBy(r => r.ExternalId) : representations.OrderByDescending(r => r.ExternalId);
177+
break;
178+
case $"{StandardSCIMRepresentationAttributes.Meta}.{StandardSCIMMetaAttributes.ResourceType}":
179+
result = order == SearchSCIMRepresentationOrders.Ascending ? representations.OrderBy(r => r.ResourceType) : representations.OrderByDescending(r => r.ResourceType);
180+
break;
181+
case $"{StandardSCIMRepresentationAttributes.Meta}.{StandardSCIMMetaAttributes.Created}":
182+
result = order == SearchSCIMRepresentationOrders.Ascending ? representations.OrderBy(r => r.Created) : representations.OrderByDescending(r => r.Created);
183+
break;
184+
case $"{StandardSCIMRepresentationAttributes.Meta}.{StandardSCIMMetaAttributes.LastModified}":
185+
result = order == SearchSCIMRepresentationOrders.Ascending ? representations.OrderBy(r => r.LastModified) : representations.OrderByDescending(r => r.LastModified);
186+
break;
187+
case $"{StandardSCIMRepresentationAttributes.Meta}.{StandardSCIMMetaAttributes.Version}":
188+
result = order == SearchSCIMRepresentationOrders.Ascending ? representations.OrderBy(r => r.Version) : representations.OrderByDescending(r => r.Version);
189+
break;
190+
case StandardSCIMRepresentationAttributes.DisplayName:
191+
result = order == SearchSCIMRepresentationOrders.Ascending ? representations.OrderBy(r => r.DisplayName) : representations.OrderByDescending(r => r.DisplayName);
192+
break;
193+
default:
194+
return null;
195+
}
196+
197+
return new PaginationResult
198+
{
199+
Query = result.Skip(startIndex <= 1 ? 0 : startIndex - 1).Take(count)
200+
};
201+
}
202+
203+
private async Task<PaginationResult> OrderByPropertyAndPaginate(
204+
IMongoQueryable<SCIMRepresentationModel> representations,
205+
SCIMAttributeExpression attrExpression,
206+
List<string> schemaNames,
207+
SearchSCIMRepresentationOrders order,
208+
int startIndex,
209+
int count)
210+
{
211+
var fullPath = attrExpression.GetFullPath();
212+
var attributes = _scimDbContext.SCIMRepresentationAttributeLst
213+
.AsQueryable()
214+
.Where(r => r.FullPath == fullPath && schemaNames.Contains(r.Namespace));
215+
var lastExpr = attrExpression.GetLastChild();
216+
switch(lastExpr.SchemaAttribute.Type)
217+
{
218+
case SCIMSchemaAttributeTypes.STRING:
219+
attributes = order == SearchSCIMRepresentationOrders.Ascending ? attributes.OrderBy(a => a.ValueString) : attributes.OrderByDescending(a => a.ValueString);
220+
break;
221+
case SCIMSchemaAttributeTypes.BOOLEAN:
222+
attributes = order == SearchSCIMRepresentationOrders.Ascending ? attributes.OrderBy(a => a.ValueBoolean) : attributes.OrderByDescending(a => a.ValueBoolean);
223+
break;
224+
case SCIMSchemaAttributeTypes.DECIMAL:
225+
attributes = order == SearchSCIMRepresentationOrders.Ascending ? attributes.OrderBy(a => a.ValueDecimal) : attributes.OrderByDescending(a => a.ValueDecimal);
226+
break;
227+
case SCIMSchemaAttributeTypes.INTEGER:
228+
attributes = order == SearchSCIMRepresentationOrders.Ascending ? attributes.OrderBy(a => a.ValueInteger) : attributes.OrderByDescending(a => a.ValueInteger);
229+
break;
230+
case SCIMSchemaAttributeTypes.DATETIME:
231+
attributes = order == SearchSCIMRepresentationOrders.Ascending ? attributes.OrderBy(a => a.ValueDateTime) : attributes.OrderByDescending(a => a.ValueDateTime);
232+
break;
233+
case SCIMSchemaAttributeTypes.BINARY:
234+
attributes = order == SearchSCIMRepresentationOrders.Ascending ? attributes.OrderBy(a => a.ValueBinary) : attributes.OrderByDescending(a => a.ValueBinary);
235+
break;
236+
case SCIMSchemaAttributeTypes.REFERENCE:
237+
attributes = order == SearchSCIMRepresentationOrders.Ascending ? attributes.OrderBy(a => a.ValueReference) : attributes.OrderByDescending(a => a.ValueReference);
238+
break;
239+
}
240+
241+
var representationIds = (await attributes
242+
.Select(a => a.RepresentationId)
243+
.Skip(startIndex <= 1 ? 0 : startIndex - 1)
244+
.Take(count)
245+
.ToListAsync())
246+
.Distinct()
247+
.ToList();
248+
return new PaginationResult
249+
{
250+
Query = representations.Where(r => representationIds.Contains(r.Id)),
251+
OrderedRepresentationIds = representationIds
252+
};
253+
}
254+
255+
private record PaginationResult
256+
{
257+
public IMongoQueryable<SCIMRepresentationModel> Query { get; set; }
258+
public List<string> OrderedRepresentationIds { get; set; }
259+
}
116260
}
117261
}

0 commit comments

Comments
 (0)