diff --git a/compose/config/config.py b/compose/config/config.py
index c5791e1533c..853da7c1c64 100644
--- a/compose/config/config.py
+++ b/compose/config/config.py
@@ -728,9 +728,22 @@ def validate_extended_service_dict(service_dict, filename, service):
 
 
 def validate_service(service_config, service_names, config_file):
+    def build_image():
+        args = sys.argv[1:]
+        if 'pull' in args:
+            return False
+
+        if '--no-build' in args:
+            return False
+
+        return True
+
     service_dict, service_name = service_config.config, service_config.name
     validate_service_constraints(service_dict, service_name, config_file)
-    validate_paths(service_dict)
+
+    if build_image():
+        # We only care about valid paths when actually building images
+        validate_paths(service_dict)
 
     validate_cpu(service_config)
     validate_ulimits(service_config)
diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py
index 82558651492..be3e28c63e4 100644
--- a/tests/acceptance/cli_test.py
+++ b/tests/acceptance/cli_test.py
@@ -748,6 +748,20 @@ def test_build_no_cache(self):
         assert BUILD_CACHE_TEXT not in result.stdout
         assert BUILD_PULL_TEXT not in result.stdout
 
+    def test_up_ignore_missing_build_directory(self):
+        self.base_dir = 'tests/fixtures/no-build'
+        result = self.dispatch(['up', '--no-build'])
+
+        assert 'alpine exited with code 0' in result.stdout
+        self.base_dir = None
+
+    def test_pull_ignore_missing_build_directory(self):
+        self.base_dir = 'tests/fixtures/no-build'
+        result = self.dispatch(['pull'])
+
+        assert 'Pulling my-alpine' in result.stderr
+        self.base_dir = None
+
     def test_build_pull(self):
         # Make sure we have the latest busybox already
         pull_busybox(self.client)
diff --git a/tests/fixtures/no-build/docker-compose.yml b/tests/fixtures/no-build/docker-compose.yml
new file mode 100644
index 00000000000..f320d17c394
--- /dev/null
+++ b/tests/fixtures/no-build/docker-compose.yml
@@ -0,0 +1,8 @@
+version: "3"
+services:
+    my-alpine:
+        image: alpine:3.12
+        container_name: alpine
+        entrypoint: 'echo It works!'
+        build:
+          context: /this/path/doesnt/exist # and we don't really care. We just want to run containers already pulled.