Merge pull request #164 from CSCfi/qa

Co-publication handling

aspnetcore/src/ApiModels/Publication/Publication.cs

@@ -283,4 +283,14 @@ public class Publication
     /// Modification time
     /// </summary>
     public DateTime? Modified { get; set; }
+
+    /// <summary>
+    /// Combined information of co-publications based on national publication data collection 
+    /// </summary>
+    public string? CoPublicationID { get; set; }
+
+    /// <summary>
+    /// Publication information sent by individual organisations related to co-publications as part of national publication data collection
+    /// </summary>
+    public List<String>? OrgPublicationIDs { get; set; }   
 }

aspnetcore/src/ApiModels/Query/GetPublicationsQueryParameters.cs

@@ -184,4 +184,16 @@ public class GetPublicationsQueryParameters : PaginationQueryParameters
     /// </summary>
     /// <see cref="Publication.Modified"/>
     public string? ModifiedTo { get; set; }
+
+    /// <summary>
+    /// If the selection is true, publication information sent by individual organisations related to co-publications as part of national publication data collection will be returned. Not returned by default.
+    /// </summary>
+    /// <see cref="Publication.ShowOrganisationPartofCoPublication"/>
+    public bool? ShowOrganisationPartofCoPublication { get; set; }
+
+    /// <summary>
+    /// If the selection is true, combined information of co-publications based on national publication data collection will be filtered out. Returned by default.
+    /// </summary>
+    /// <see cref="Publication.HideCoPublications"/>
+    public bool? HideCoPublications { get; set; }
 }

aspnetcore/src/ElasticService/ElasticSearchExtensions.cs

@@ -36,8 +36,10 @@ private static ConnectionSettings GetConnectionSettings(IConfiguration configura
     {
         var elasticSearchClusterUrl = configuration["ElasticSearch:Url"] ?? throw new InvalidOperationException("ElasticSearch url missing.");
 
+        // https://www.elastic.co/guide/en/elasticsearch/client/net-api/7.17/configuration-options.html
         var connectionSettings = new ConnectionSettings(new Uri(elasticSearchClusterUrl))
-            .DefaultFieldNameInferrer(i => i); // This forces elastic to store .Net objects using type names, instead of camel casing. This enables using nameof when referring to fields.
+            .DefaultFieldNameInferrer(i => i) // This forces elastic to store .Net objects using type names, instead of camel casing. This enables using nameof when referring to fields.
+            .MaximumRetries(5);
 
         if (Debugger.IsAttached)
         {

aspnetcore/src/ElasticService/ElasticSearchIndexService.cs

@@ -8,7 +8,7 @@ public class ElasticSearchIndexService : IElasticSearchIndexService
     private readonly IElasticClient _elasticClient;
     private readonly ILogger<ElasticSearchIndexService> _logger;
 
-    private const int BatchSize = 2000;
+    private const int BatchSize = 1500;
 
     public ElasticSearchIndexService(IElasticClient elasticClient, ILogger<ElasticSearchIndexService> logger)
     {

aspnetcore/src/ElasticService/ElasticSearchQueryGenerators/PublicationQueryGenerator.cs

@@ -150,6 +150,20 @@ private static IEnumerable<Func<QueryContainerDescriptor<Publication>, QueryCont
     {
         var filters = new List<Func<QueryContainerDescriptor<Publication>, QueryContainer>>();
 
+        if (parameters.ShowOrganisationPartofCoPublication is null || (bool)parameters.ShowOrganisationPartofCoPublication == false)
+        {
+            filters.Add(t =>
+                t.Term(s => s.Field(f => f.IsOrgPublication)
+                    .Value(false)));
+        }
+
+        if (parameters.HideCoPublications is not null && (bool)parameters.HideCoPublications == true)
+        {
+            filters.Add(t =>
+                t.Term(s => s.Field(f => f.IsCoPublication)
+                    .Value(false)));
+        }
+
         if (parameters.CreatedFrom is not null)
         {
             filters.Add(x => x

aspnetcore/src/ElasticService/SearchParameters/PublicationSearchParameters.cs

@@ -55,4 +55,8 @@ public class PublicationSearchParameters
     public string? ModifiedFrom { get; set; }
 
     public string? ModifiedTo { get; set; }
+
+    public bool? ShowOrganisationPartofCoPublication { get; set; }
+
+    public bool? HideCoPublications { get; set; }
 }

aspnetcore/src/Indexer/Indexer.cs

@@ -148,7 +148,7 @@ private async Task IndexEntities(string indexName, IIndexRepository repository,
                 */
 
                 int skipAmount = 0;
-                int takeAmount = 50000;
+                int takeAmount = 30000;
                 int numOfResults = 0;
                 int processedCount = 0;
 

aspnetcore/src/Repositories/Maps/PublicationProfile.cs

@@ -28,6 +28,7 @@ public PublicationProfile()
             .ForMember(dst => dst.ApcPaymentYear, opt =>opt.MapFrom(src => (DateTime?)(src.ApcPaymentYear.HasValue ? new DateTime(src.ApcPaymentYear.Value,1,1,0,0,0,DateTimeKind.Utc) : null)))
             .ForMember(dst => dst.AuthorsText, opt => opt.MapFrom(src => src.AuthorsText))
             .ForMember(dst => dst.DatabaseContributions, opt => opt.MapFrom(src => src.FactContributions))
+            .ForMember(dst => dst.OrgPublicationDatabaseContributionDTOs, opt => opt.MapFrom(src => src.InverseDimPublicationNavigation))
             .ForMember(dst => dst.Format, opt => opt.MapFrom(src => src.PublicationTypeCode2Navigation))
             .ForMember(dst => dst.ParentPublicationType, opt => opt.MapFrom(src => src.ParentPublicationTypeCodeNavigation))
             .ForMember(dst => dst.DatabasePeerReviewed, opt => opt.MapFrom(src => src.PeerReviewed))
@@ -55,8 +56,11 @@ public PublicationProfile()
             .ForMember(dst => dst.Doi, opt => opt.MapFrom(src => src.Doi))
             .ForMember(dst => dst.DoiHandle, opt => opt.MapFrom(src => src.DoiHandle))
             .ForMember(dst => dst.FieldsOfScience, opt => opt.MapFrom(src => src.FactDimReferencedataFieldOfSciences.Select(f => f.DimReferencedata)))
+            .ForMember(dst => dst.OrgPublicationFieldsOfScienceDTOs, opt => opt.MapFrom(src => src.InverseDimPublicationNavigation))
             .ForMember(dst => dst.FieldsOfArt, opt => opt.MapFrom(src => src.DimReferencedataNavigation))
+            .ForMember(dst => dst.OrgPublicationFieldsOfArtDTOs, opt => opt.MapFrom(src => src.InverseDimPublicationNavigation))
             .ForMember(dst => dst.Keywords, opt => opt.MapFrom(src => src.DimKeywords))
+            .ForMember(dst => dst.OrgPublicationKeywordDTOs, opt => opt.MapFrom(src => src.InverseDimPublicationNavigation))
             .ForMember(dst => dst.InternationalPublication, opt => opt.MapFrom(src => src.InternationalPublication != 9 ? src.InternationalPublication == 1 : (bool?)null)) // 0 = kotim. 1 ulkom. 9 = ei tietoa.
             .ForMember(dst => dst.Country, opt => opt.MapFrom(src => src.PublicationCountryCodeNavigation))
             .ForMember(dst => dst.Language, opt => opt.MapFrom(src => src.LanguageCodeNavigation))
@@ -69,6 +73,7 @@ public PublicationProfile()
             .ForMember(dst => dst.Preprint, opt => opt.MapFrom(src => src.DimLocallyReportedPubInfos.Where(i => i.SelfArchivedType == PreprintType)))
             .ForMember(dst => dst.SelfArchived, opt => opt.MapFrom(src => src.DimLocallyReportedPubInfos.Where(i => i.SelfArchivedType == SelfArchivedType)))
             .ForMember(dst => dst.ArtPublicationTypeCategory, opt => opt.MapFrom(src => src.DimReferencedata))
+            .ForMember(dst => dst.OrgPublicationArtPublicatonTypeCategoryDTOs, opt => opt.MapFrom(src => src.InverseDimPublicationNavigation))
             .ForMember(dst => dst.Abstract, opt => opt.MapFrom(src => src.Abstract))
             .ForMember(dst => dst.Created, opt => opt.MapFrom(src => src.Created))
             .ForMember(dst => dst.Modified, opt => opt.MapFrom(src => src.Modified))
@@ -78,7 +83,10 @@ public PublicationProfile()
             .ForMember(dst => dst.Authors, opt => opt.Ignore())  // Handled during in memory operations in the index repository
             .ForMember(dst => dst.PeerReviewed, opt => opt.Ignore())  // Handled during in memory operations in the index repository
             .ForMember(dst => dst.ParentPublication, opt => opt.Ignore())  // Handled during in memory operations in the index repository
-            ;
+            .ForMember(dst => dst.IsOrgPublication, opt => opt.MapFrom(src => src.DimPublicationId != null && src.DimPublicationId > 0)) // Publication is an organization publication, when DimPublicationId references co-publication. This property is used in query filter.
+            .ForMember(dst => dst.IsCoPublication, opt => opt.MapFrom(src => src.InverseDimPublicationNavigation.Count > 0)) // Publication is a co-publication, when InverseDimPublicationNavigation references one or more organization publications. This property is used in query filter.
+            .ForMember(dst => dst.CoPublicationID, opt => opt.MapFrom(src => src.DimPublicationId != null && src.DimPublicationId > 0  ? src.DimPublicationNavigation.PublicationId : null))
+            .ForMember(dst => dst.OrgPublicationIDs, opt => opt.MapFrom(src => src.InverseDimPublicationNavigation.Select(t => t.PublicationId)));
 
         CreateProjection<DimReferencedatum, ReferenceData>()
             .AddTransform<string?>(s => string.IsNullOrWhiteSpace(s) ? null : s)
@@ -133,6 +141,26 @@ public PublicationProfile()
             .ForMember(dst => dst.ArtPublicationRole, opt => opt.MapFrom(src => src.DimReferencedataActorRole))
             .ForMember(dst => dst.ContributionType, opt => opt.MapFrom(src => src.ContributionType));
 
+        CreateProjection<DatabaseContext.Entities.DimPublication, OrgPublicationDatabaseContributionDTO>()
+            .ForMember(dst => dst.Id, opt => opt.MapFrom(src => src.Id))
+            .ForMember(dst => dst.DatabaseContributions, opt => opt.MapFrom(src => src.FactContributions));
+
+        CreateProjection<DatabaseContext.Entities.DimPublication, OrgPublicationKeywordDTO>()
+            .ForMember(dst => dst.Id, opt => opt.MapFrom(src => src.Id))
+            .ForMember(dst => dst.Keywords, opt => opt.MapFrom(src => src.DimKeywords));
+
+        CreateProjection<DatabaseContext.Entities.DimPublication, OrgPublicationArtPublicatonTypeCategoryDTO>()
+            .ForMember(dst => dst.Id, opt => opt.MapFrom(src => src.Id))
+            .ForMember(dst => dst.ArtPublicationTypeCategories, opt => opt.MapFrom(src => src.DimReferencedata));
+
+        CreateProjection<DatabaseContext.Entities.DimPublication, OrgPublicationFieldsOfScienceDTO>()
+            .ForMember(dst => dst.Id, opt => opt.MapFrom(src => src.Id))
+            .ForMember(dst => dst.FieldsOfScience, opt => opt.MapFrom(src => src.FactDimReferencedataFieldOfSciences.Select(f => f.DimReferencedata)));
+
+        CreateProjection<DatabaseContext.Entities.DimPublication, OrgPublicationFieldsOfArtDTO>()
+            .ForMember(dst => dst.Id, opt => opt.MapFrom(src => src.Id))
+            .ForMember(dst => dst.FieldsOfArt, opt => opt.MapFrom(src => src.DimReferencedataNavigation));
+
         CreateProjection<DimName, Name>()
             .AddTransform<string?>(s => string.IsNullOrWhiteSpace(s) ? null : s)
             .ForMember(dst => dst.NameId, opt => opt.MapFrom(src => src.Id))

aspnetcore/src/Repositories/PublicationIndexRepository.cs

@@ -3,7 +3,6 @@
 using CSC.PublicApi.DatabaseContext;
 using CSC.PublicApi.Service.Models;
 using CSC.PublicApi.Service.Models.Publication;
-using Elasticsearch.Net;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Caching.Memory;
 using Publication = CSC.PublicApi.Service.Models.Publication.Publication;
@@ -81,7 +80,54 @@ public override List<object> PerformInMemoryOperations(List<object> entities)
     public override object PerformInMemoryOperation(object entity)
     {
         Publication publication = (Publication)entity;
-
+
+        // Add data from OrgPublication DTOs, which hold data collected from publications related to a co-publication (yhteisjulkaisu/osajulkaisu)
+        if (publication.OrgPublicationDatabaseContributionDTOs != null && publication.OrgPublicationDatabaseContributionDTOs.Count > 0)
+        {
+            List<FactContribution> databaseContributions = new();
+            foreach (OrgPublicationDatabaseContributionDTO op in publication.OrgPublicationDatabaseContributionDTOs.AsEnumerable().ToList())
+            {
+                databaseContributions.AddRange(op.DatabaseContributions);
+            }
+            publication.DatabaseContributions.AddRange(databaseContributions.DistinctBy(x => x.Name));
+        }
+        if (publication.OrgPublicationKeywordDTOs != null && publication.OrgPublicationKeywordDTOs.Count > 0)
+        {
+            List<Keyword> keywords = new();
+            foreach (OrgPublicationKeywordDTO kw in publication.OrgPublicationKeywordDTOs.AsEnumerable().ToList())
+            {
+                keywords.AddRange(kw.Keywords);
+            }
+            publication.Keywords.AddRange(keywords.DistinctBy(x => x.Value));
+        }
+        if (publication.OrgPublicationArtPublicatonTypeCategoryDTOs != null && publication.OrgPublicationArtPublicatonTypeCategoryDTOs.Count > 0)
+        {
+            List<ReferenceData> artPublicationTypeCategories = new();
+            foreach (OrgPublicationArtPublicatonTypeCategoryDTO ap in publication.OrgPublicationArtPublicatonTypeCategoryDTOs.AsEnumerable().ToList())
+            {
+                artPublicationTypeCategories.AddRange(ap.ArtPublicationTypeCategories);
+            }
+            publication.ArtPublicationTypeCategory.AddRange(artPublicationTypeCategories.DistinctBy(x => x.Code));
+        }
+        if (publication.OrgPublicationFieldsOfScienceDTOs != null && publication.OrgPublicationFieldsOfScienceDTOs.Count > 0)
+        {
+            List<ReferenceData> fieldsOfScience = new();
+            foreach (OrgPublicationFieldsOfScienceDTO fs in publication.OrgPublicationFieldsOfScienceDTOs.AsEnumerable().ToList())
+            {
+                fieldsOfScience.AddRange(fs.FieldsOfScience);
+            }
+            publication.FieldsOfScience.AddRange(fieldsOfScience.DistinctBy(x => x.Code));
+        }
+        if (publication.OrgPublicationFieldsOfArtDTOs != null && publication.OrgPublicationFieldsOfArtDTOs.Count > 0)
+        {
+            List<ReferenceData> fieldsOfArt = new();
+            foreach (OrgPublicationFieldsOfArtDTO fa in publication.OrgPublicationFieldsOfArtDTOs.AsEnumerable().ToList())
+            {
+                fieldsOfArt.AddRange(fa.FieldsOfArt);
+            }
+            publication.FieldsOfArt.AddRange(fieldsOfArt.DistinctBy(x => x.Code));
+        }
+
         HandleIssnAndIsbn(publication);
         HandleEmptyCollections(publication);
         HandleOrganizations(publication);

aspnetcore/src/Service.Models/Publication/OrgPublicationArtPublicatonTypeCategoryDTO.cs

@@ -0,0 +1,9 @@
+namespace CSC.PublicApi.Service.Models.Publication;
+
+// DTO used when collecting ArtPublicationTypeCategories from OrgPublications (yhteisjulkaisu/osajulkaisu).
+public class OrgPublicationArtPublicatonTypeCategoryDTO
+{
+    public string? Id { get; set; }
+
+    public List<ReferenceData>? ArtPublicationTypeCategories { get; set; }
+}

aspnetcore/src/Service.Models/Publication/OrgPublicationDatabaseContributionDTO.cs

@@ -0,0 +1,9 @@
+namespace CSC.PublicApi.Service.Models.Publication;
+
+// DTO used when collecting DatabaseContributions from OrgPublications (yhteisjulkaisu/osajulkaisu).
+public class OrgPublicationDatabaseContributionDTO
+{
+    public string? Id { get; set; }
+
+    public List<FactContribution>? DatabaseContributions { get; set; }
+}

aspnetcore/src/Service.Models/Publication/OrgPublicationFieldsOfArtDTO.cs

@@ -0,0 +1,9 @@
+namespace CSC.PublicApi.Service.Models.Publication;
+
+// DTO used when collecting fields of art from OrgPublications (yhteisjulkaisu/osajulkaisu).
+public class OrgPublicationFieldsOfArtDTO
+{
+    public string? Id { get; set; }
+
+    public List<ReferenceData>? FieldsOfArt { get; set; }
+}

aspnetcore/src/Service.Models/Publication/OrgPublicationFieldsOfScienceDTO copy.cs

@@ -0,0 +1,9 @@
+namespace CSC.PublicApi.Service.Models.Publication;
+
+// DTO used when collecting fields of science from OrgPublications (yhteisjulkaisu/osajulkaisu).
+public class OrgPublicationFieldsOfScienceDTO
+{
+    public string? Id { get; set; }
+
+    public List<ReferenceData>? FieldsOfScience { get; set; }
+}

aspnetcore/src/Service.Models/Publication/OrgPublicationKeywordDTO.cs

@@ -0,0 +1,9 @@
+namespace CSC.PublicApi.Service.Models.Publication;
+
+// DTO used when collecting Keywords from OrgPublications (yhteisjulkaisu/osajulkaisu).
+public class OrgPublicationKeywordDTO
+{
+    public string? Id { get; set; }
+
+    public List<Keyword>? Keywords { get; set; }
+}

aspnetcore/src/Service.Models/Publication/Publication.cs

@@ -259,6 +259,26 @@ public class Publication
     /// </summary>
     public DateTime? Modified { get; set; }
 
+    /// <summary>
+    /// Onko julkaisu yhteisjulkaisu
+    /// </summary>
+    public bool IsCoPublication { get; set; }
+
+    /// <summary>
+    /// Onko julkaisu osajulkaisu
+    /// </summary>
+    public bool IsOrgPublication { get; set; }
+
+    /// <summary>
+    /// Osajulkaisun yhteisjulkaisu
+    /// </summary>
+    public string? CoPublicationID { get; set; }
+
+    /// <summary>
+    /// Yhteisjulkaisuun liittyvät osajulkaisut
+    /// </summary>
+    public List<String>? OrgPublicationIDs { get; set; }   
+
     [Ignore]
     public string? Isbn1 { get; set; }
 
@@ -274,6 +294,21 @@ public class Publication
     [Ignore]
     public List<FactContribution>? DatabaseContributions { get; set; }
 
+    [Ignore]
+    public List<OrgPublicationDatabaseContributionDTO>? OrgPublicationDatabaseContributionDTOs { get; set; }
+
+    [Ignore]
+    public List<OrgPublicationKeywordDTO>? OrgPublicationKeywordDTOs { get; set; }
+
+    [Ignore]
+    public List<OrgPublicationArtPublicatonTypeCategoryDTO>? OrgPublicationArtPublicatonTypeCategoryDTOs { get; set; }
+
+    [Ignore]
+    public List<OrgPublicationFieldsOfScienceDTO>? OrgPublicationFieldsOfScienceDTOs { get; set; }
+
+    [Ignore]
+    public List<OrgPublicationFieldsOfArtDTO>? OrgPublicationFieldsOfArtDTOs { get; set; }
+
     [Ignore]
     public bool? DatabasePeerReviewed { get; set; }