diff --git a/addons/web_editor/static/src/components/media_dialog/image_selector.js b/addons/web_editor/static/src/components/media_dialog/image_selector.js
index b67f49aff66..a9cf4dc8f27 100644
--- a/addons/web_editor/static/src/components/media_dialog/image_selector.js
+++ b/addons/web_editor/static/src/components/media_dialog/image_selector.js
@@ -68,9 +68,8 @@ export class ImageSelector extends FileSelector {
     }
 
     get canLoadMore() {
-        if (this.state.searchService === 'all') {
-            return super.canLoadMore || (this.state.libraryResults && this.state.libraryMedia.length < this.state.libraryResults);
-        } else if (this.state.searchService === 'media-library') {
+        // The user can load more library media only when the filter is set.
+        if (this.state.searchService === 'media-library') {
             return this.state.libraryResults && this.state.libraryMedia.length < this.state.libraryResults;
         }
         return super.canLoadMore;
@@ -182,7 +181,10 @@ export class ImageSelector extends FileSelector {
 
     async loadMore(...args) {
         await super.loadMore(...args);
-        if (!this.props.useMediaLibrary) {
+        if (!this.props.useMediaLibrary
+            // The user can load more library media only when the filter is set.
+            || this.state.searchService !== 'media-library'
+        ) {
             return;
         }
         return this.keepLastLibraryMedia.add(this.fetchLibraryMedia(this.state.libraryMedia.length)).then(({ media }) => {