[FIXED] Why filtering does not work with Spring Boot and MongoDB

Issue

Why filtering by import_created_at does not work?

I have a Spring Boot Application and MongoDB as database.

I have Mongo collection items and 2 documents there:

{
  "_id": {
    "product_id": "11",
    "contract_id": {
      "$numberLong": "1"
    }
  },
  "contract_id": {
    "$numberLong": "1"
  },
  "update": {
    "import_created_at": {
      "$numberLong": "1661784425743"
    },
    "product_id": "11",
    "status": "COMPLETED"
  },
  "_class": "com.documents.ItemDoc"
}


{
  "_id": {
    "product_id": "22",
    "contract_id": {
      "$numberLong": "1"
    }
  },
  "contract_id": {
    "$numberLong": "1"
  },
  "update": {
    "import_created_at": {
      "$numberLong": "1661784425999"
    },
    "product_id": "22",
    "status": "COMPLETED"
  },
  "_class": "com.documents.ItemDoc"
}

I have entity classes for items:

@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
@Document(collection = "items")
public class ItemEntity {

    @Id
    private ItemId id;

    @Indexed
    @Field(name = "contract_id")
    private Long contractId;

    @Field(name = "product_id")
    private String productId;

    @Field(name = "update")
    private Update update;
}

and

@Data
   @Builder
   @NoArgsConstructor
   @AllArgsConstructor
    public class Update {
    
        @Field(name = "import_created_at")
        private Long importCreatedAt;
    
        @Field(name = "product_id")
        private String productId;
    
        @Field(name = "status")
        private String status;
    }

and

@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class ItemId implements Serializable {

    @Field(name = "product_id")
    private String productId;

    @Field(name = "contract_id")
    private Long sellerContractId;
}

and

@Builder
@Data
@AllArgsConstructor
@NoArgsConstructor
@Document
public class ItemEntityFacet {
    private List<ItemEntity> itemAggregationResult;
}

and

@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class ItemFilters {

    private List<String> productIds;
    private List<Long> lastUpdates;
}

and

@Getter
@NoArgsConstructor
public class ItemMatchOperation {

    private Map<String, Object> conditions;

    private AggregationOperation matchOperation;

    @Builder
    public OfferMatchOperation(Map<String, Object> conditions) {
        this.conditions = conditions;
    }

    private ItemMatchOperation(Map<String, Object> conditions, AggregationOperation operation) {
        this.conditions = conditions;
        this.matchOperation = operation;
    }

    public static class ItemMatchOperationBuilder {
        private AggregationOperation facetOperation;

        public ItemMatchOperation build() {

            Validate.notEmpty(conditions, "conditions cannot be null or empty!");
            buildOperation();
            return new ItemMatchOperation(conditions, facetOperation);
        }
        private void buildOperation() {
            facetOperation = context ->  new Document("$match",buildMatchConditions());
        }

        private Document buildMatchConditions() {
            Document document = new Document();
            conditions.entrySet().stream().forEach(entry -> document.append(entry.getKey(), entry.getValue()));
            return document;
        }
    }
}

And I have repository class:

@Slf4j
@Repository
@RequiredArgsConstructor
public class ItemsRepositoryImpl implements ItemsRepository {

private final ReactiveMongoTemplate mongoTemplate;

public Flux<ItemEntity> findAllByFilters(Long contractId, ItemFilters filters, Pageable pageable) {
        Aggregation aggregation = getAggregation(contractId, filters, pageable);
        return mongoTemplate.aggregate(aggregation, "items",
                        ItemEntityFacet.class)
                .map(ItemEntityFacet::getItemAggregationResult)
                .filter(itemEntities -> !itemEntities.isEmpty())
                .flatMap(Flux::fromIterable);
    }

  private Aggregation getAggregation(Long contractId, OffersListingFilters filters, Pageable pageable) {

        List<AggregationOperation> aggregationOperationList = new ArrayList<>();

        if (CollectionUtils.isNotEmpty(filters.getProductIds())) {
            aggregationOperationList.add(getProductIdMatchOperation(contractId, 
            filters.getProductIds()));
        }
        if (CollectionUtils.isNotEmpty(filters.getLastUpdates())) {
            aggregationOperationList.add(getLastUpdatesMatchOperation(filters.getLastUpdates()));
        }
        aggregationOperationList.add(getContractMatchOperation(contractId));
        aggregationOperationList.add(getFacetOperation(pageable));

        return Aggregation.newAggregation(aggregationOperationList);
    }


   private AggregationOperation getProductIdMatchOperation(Long contractId, List<String> productIds) {
        return ItemMatchOperation.builder()
                .conditions(Map.of("_id", new Document("$in", getDocIdsToMatch(contractId, skus))))
                .build()
                .getMatchOperation();
    }

   private List<Document> getDocIdsToMatch(Long contractId, List<String> productIds) {
        return productIds.stream().map(productId -> new Document("product_id", productId)
                        .append("contract_id", contractId))
                .collect(Collectors.toList());
    }

    private AggregationOperation getLastUpdatesMatchOperation(List<Long> lastUpdates) {
        return ItemMatchOperation.builder()
                .conditions(Map.of("update", new Document("$in", getDocToMatchLastUpdates(lastUpdates))))
                .build()
                .getMatchOperation();
    }


  private List<Document> getDocToMatchLastUpdates(List<Long> lastUpdates) {
        return lastUpdates.stream().map(lastUpdate -> new Document("import_created_at", lastUpdate))
                .collect(Collectors.toList());
    }

 private AggregationOperation getContractMatchOperation(Long contractId) {
        return ItemMatchOperation.builder()
                .conditions(Map.of("contract_id", contractId))
                .build()
                .getMatchOperation();
    }

...
}

When I call the repo method findAllByFilters without filters (only by contractId = 1) it returns 2 documents as expected

When I add productId as filter and get it by contractId = 1 and productId = 11 it returns one doc as expected

BUT when I call with contractId = 1 and lastUpdate = 1661784425743 it returns nothing but should return 1st document. What is wrong here?

Solution

Per the comments, the code above is generating a query similar to:

{ 
  "aggregate" : "collection", 
  "pipeline" : [
    { "$match" : 
      { 
        "update": { "$in" : [{ "import_created_at" : 1661784425743}]
      }
    }
  ]
}

This syntax attempts to do perform an equality match on the whole embedded document requiring an exact match of the entire filter (including field order). You can see a demonstration of that in this Mongo Playground, where the second document is the one that doesn't match for interesting reasons.

Instead, you will want to use dot notation to query on a nested field. The syntax for that pipeline looks something like this:

[
  {
    $match: {
      "update.import_created_at": {
        $in: [
          166178442574
        ]
      }
    }
  }
]

And the associated Mongo Playground example shows both of the documents getting returned as expected.

Answered By - user20042973
Answer Checked By - Pedro (JavaFixing Volunteer)