diff --git a/internal/querynode/collection_replica.go b/internal/querynode/collection_replica.go
index 5a3001fd09..d4f39d628c 100644
--- a/internal/querynode/collection_replica.go
+++ b/internal/querynode/collection_replica.go
@@ -111,6 +111,7 @@ func (colReplica *collectionReplicaImpl) removeCollection(collectionID UniqueID)
 		if col.ID() == collectionID {
 			for _, p := range *col.Partitions() {
 				for _, s := range *p.Segments() {
+					deleteSegment(colReplica.segments[s.ID()])
 					delete(colReplica.segments, s.ID())
 				}
 			}
@@ -202,6 +203,7 @@ func (colReplica *collectionReplicaImpl) removePartition(collectionID UniqueID,
 	for _, p := range *collection.Partitions() {
 		if p.Tag() == partitionTag {
 			for _, s := range *p.Segments() {
+				deleteSegment(colReplica.segments[s.ID()])
 				delete(colReplica.segments, s.ID())
 			}
 		} else {