dulwich/tests/compat/test_client.py

   1 # test_client.py -- Compatibilty tests for git client.
   2 # Copyright (C) 2010 Google, Inc.
   3 #
   4 # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
   5 # General Public License as public by the Free Software Foundation; version 2.0
   6 # or (at your option) any later version. You can redistribute it and/or
   7 # modify it under the terms of either of these two licenses.
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 #
  15 # You should have received a copy of the licenses; if not, see
  16 # <http://www.gnu.org/licenses/> for a copy of the GNU General Public License
  17 # and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
  18 # License, Version 2.0.
  19 #
  20
  21 """Compatibilty tests between the Dulwich client and the cgit server."""
  22
  23 import copy
  24 from io import BytesIO
  25 import os
  26 import select
  27 import signal
  28 import subprocess
  29 import sys
  30 import tarfile
  31 import tempfile
  32 import threading
  33
  34 try:
  35     from urlparse import unquote
  36 except ImportError:
  37     from urllib.parse import unquote
  38
  39
  40 try:
  41     import BaseHTTPServer
  42     import SimpleHTTPServer
  43 except ImportError:
  44     import http.server
  45     BaseHTTPServer = http.server
  46     SimpleHTTPServer = http.server
  47
  48 from dulwich import (
  49     client,
  50     errors,
  51     file,
  52     index,
  53     protocol,
  54     objects,
  55     repo,
  56     )
  57 from dulwich.tests import (
  58     SkipTest,
  59     expectedFailure,
  60     )
  61 from dulwich.tests.compat.utils import (
  62     CompatTestCase,
  63     check_for_daemon,
  64     import_repo_to_dir,
  65     rmtree_ro,
  66     run_git_or_fail,
  67     _DEFAULT_GIT,
  68     )
  69
  70
  71 if sys.platform == 'win32':
  72     import ctypes
  73
  74
  75 class DulwichClientTestBase(object):
  76     """Tests for client/server compatibility."""
  77
  78     def setUp(self):
  79         self.gitroot = os.path.dirname(
  80                 import_repo_to_dir('server_new.export').rstrip(os.sep))
  81         self.dest = os.path.join(self.gitroot, 'dest')
  82         file.ensure_dir_exists(self.dest)
  83         run_git_or_fail(['init', '--quiet', '--bare'], cwd=self.dest)
  84
  85     def tearDown(self):
  86         rmtree_ro(self.gitroot)
  87
  88     def assertDestEqualsSrc(self):
  89         repo_dir = os.path.join(self.gitroot, 'server_new.export')
  90         dest_repo_dir = os.path.join(self.gitroot, 'dest')
  91         with repo.Repo(repo_dir) as src:
  92             with repo.Repo(dest_repo_dir) as dest:
  93                 self.assertReposEqual(src, dest)
  94
  95     def _client(self):
  96         raise NotImplementedError()
  97
  98     def _build_path(self):
  99         raise NotImplementedError()
 100
 101     def _do_send_pack(self):
 102         c = self._client()
 103         srcpath = os.path.join(self.gitroot, 'server_new.export')
 104         with repo.Repo(srcpath) as src:
 105             sendrefs = dict(src.get_refs())
 106             del sendrefs[b'HEAD']
 107             c.send_pack(self._build_path('/dest'), lambda _: sendrefs,
 108                         src.object_store.generate_pack_data)
 109
 110     def test_send_pack(self):
 111         self._do_send_pack()
 112         self.assertDestEqualsSrc()
 113
 114     def test_send_pack_nothing_to_send(self):
 115         self._do_send_pack()
 116         self.assertDestEqualsSrc()
 117         # nothing to send, but shouldn't raise either.
 118         self._do_send_pack()
 119
 120     def test_send_without_report_status(self):
 121         c = self._client()
 122         c._send_capabilities.remove(b'report-status')
 123         srcpath = os.path.join(self.gitroot, 'server_new.export')
 124         with repo.Repo(srcpath) as src:
 125             sendrefs = dict(src.get_refs())
 126             del sendrefs[b'HEAD']
 127             c.send_pack(self._build_path('/dest'), lambda _: sendrefs,
 128                         src.object_store.generate_pack_data)
 129             self.assertDestEqualsSrc()
 130
 131     def make_dummy_commit(self, dest):
 132         b = objects.Blob.from_string(b'hi')
 133         dest.object_store.add_object(b)
 134         t = index.commit_tree(dest.object_store, [(b'hi', b.id, 0o100644)])
 135         c = objects.Commit()
 136         c.author = c.committer = b'Foo Bar <foo@example.com>'
 137         c.author_time = c.commit_time = 0
 138         c.author_timezone = c.commit_timezone = 0
 139         c.message = b'hi'
 140         c.tree = t
 141         dest.object_store.add_object(c)
 142         return c.id
 143
 144     def disable_ff_and_make_dummy_commit(self):
 145         # disable non-fast-forward pushes to the server
 146         dest = repo.Repo(os.path.join(self.gitroot, 'dest'))
 147         run_git_or_fail(['config', 'receive.denyNonFastForwards', 'true'],
 148                         cwd=dest.path)
 149         commit_id = self.make_dummy_commit(dest)
 150         return dest, commit_id
 151
 152     def compute_send(self, src):
 153         sendrefs = dict(src.get_refs())
 154         del sendrefs[b'HEAD']
 155         return sendrefs, src.object_store.generate_pack_data
 156
 157     def test_send_pack_one_error(self):
 158         dest, dummy_commit = self.disable_ff_and_make_dummy_commit()
 159         dest.refs[b'refs/heads/master'] = dummy_commit
 160         repo_dir = os.path.join(self.gitroot, 'server_new.export')
 161         with repo.Repo(repo_dir) as src:
 162             sendrefs, gen_pack = self.compute_send(src)
 163             c = self._client()
 164             try:
 165                 c.send_pack(self._build_path('/dest'),
 166                             lambda _: sendrefs, gen_pack)
 167             except errors.UpdateRefsError as e:
 168                 self.assertEqual('refs/heads/master failed to update',
 169                                  e.args[0])
 170                 self.assertEqual({b'refs/heads/branch': b'ok',
 171                                   b'refs/heads/master': b'non-fast-forward'},
 172                                  e.ref_status)
 173
 174     def test_send_pack_multiple_errors(self):
 175         dest, dummy = self.disable_ff_and_make_dummy_commit()
 176         # set up for two non-ff errors
 177         branch, master = b'refs/heads/branch', b'refs/heads/master'
 178         dest.refs[branch] = dest.refs[master] = dummy
 179         repo_dir = os.path.join(self.gitroot, 'server_new.export')
 180         with repo.Repo(repo_dir) as src:
 181             sendrefs, gen_pack = self.compute_send(src)
 182             c = self._client()
 183             try:
 184                 c.send_pack(self._build_path('/dest'), lambda _: sendrefs,
 185                             gen_pack)
 186             except errors.UpdateRefsError as e:
 187                 self.assertIn(
 188                         str(e),
 189                         ['{0}, {1} failed to update'.format(
 190                             branch.decode('ascii'), master.decode('ascii')),
 191                          '{1}, {0} failed to update'.format(
 192                              branch.decode('ascii'), master.decode('ascii'))])
 193                 self.assertEqual({branch: b'non-fast-forward',
 194                                   master: b'non-fast-forward'},
 195                                  e.ref_status)
 196
 197     def test_archive(self):
 198         c = self._client()
 199         f = BytesIO()
 200         c.archive(self._build_path('/server_new.export'), b'HEAD', f.write)
 201         f.seek(0)
 202         tf = tarfile.open(fileobj=f)
 203         self.assertEqual(['baz', 'foo'], tf.getnames())
 204
 205     def test_fetch_pack(self):
 206         c = self._client()
 207         with repo.Repo(os.path.join(self.gitroot, 'dest')) as dest:
 208             result = c.fetch(self._build_path('/server_new.export'), dest)
 209             for r in result.refs.items():
 210                 dest.refs.set_if_equals(r[0], None, r[1])
 211             self.assertDestEqualsSrc()
 212
 213     def test_repeat(self):
 214         c = self._client()
 215         with repo.Repo(os.path.join(self.gitroot, 'dest')) as dest:
 216             result = c.fetch(self._build_path('/server_new.export'), dest)
 217             for r in result.refs.items():
 218                 dest.refs.set_if_equals(r[0], None, r[1])
 219             self.assertDestEqualsSrc()
 220             result = c.fetch(self._build_path('/server_new.export'), dest)
 221             for r in result.refs.items():
 222                 dest.refs.set_if_equals(r[0], None, r[1])
 223             self.assertDestEqualsSrc()
 224
 225     def test_incremental_fetch_pack(self):
 226         self.test_fetch_pack()
 227         dest, dummy = self.disable_ff_and_make_dummy_commit()
 228         dest.refs[b'refs/heads/master'] = dummy
 229         c = self._client()
 230         repo_dir = os.path.join(self.gitroot, 'server_new.export')
 231         with repo.Repo(repo_dir) as dest:
 232             result = c.fetch(self._build_path('/dest'), dest)
 233             for r in result.refs.items():
 234                 dest.refs.set_if_equals(r[0], None, r[1])
 235             self.assertDestEqualsSrc()
 236
 237     def test_fetch_pack_no_side_band_64k(self):
 238         c = self._client()
 239         c._fetch_capabilities.remove(b'side-band-64k')
 240         with repo.Repo(os.path.join(self.gitroot, 'dest')) as dest:
 241             result = c.fetch(self._build_path('/server_new.export'), dest)
 242             for r in result.refs.items():
 243                 dest.refs.set_if_equals(r[0], None, r[1])
 244             self.assertDestEqualsSrc()
 245
 246     def test_fetch_pack_zero_sha(self):
 247         # zero sha1s are already present on the client, and should
 248         # be ignored
 249         c = self._client()
 250         with repo.Repo(os.path.join(self.gitroot, 'dest')) as dest:
 251             result = c.fetch(
 252                 self._build_path('/server_new.export'), dest,
 253                 lambda refs: [protocol.ZERO_SHA])
 254             for r in result.refs.items():
 255                 dest.refs.set_if_equals(r[0], None, r[1])
 256
 257     def test_send_remove_branch(self):
 258         with repo.Repo(os.path.join(self.gitroot, 'dest')) as dest:
 259             dummy_commit = self.make_dummy_commit(dest)
 260             dest.refs[b'refs/heads/master'] = dummy_commit
 261             dest.refs[b'refs/heads/abranch'] = dummy_commit
 262             sendrefs = dict(dest.refs)
 263             sendrefs[b'refs/heads/abranch'] = b"00" * 20
 264             del sendrefs[b'HEAD']
 265
 266             def gen_pack(have, want, ofs_delta=False):
 267                 return 0, []
 268             c = self._client()
 269             self.assertEqual(dest.refs[b"refs/heads/abranch"], dummy_commit)
 270             c.send_pack(
 271                 self._build_path('/dest'), lambda _: sendrefs, gen_pack)
 272             self.assertFalse(b"refs/heads/abranch" in dest.refs)
 273
 274     def test_get_refs(self):
 275         c = self._client()
 276         refs = c.get_refs(self._build_path('/server_new.export'))
 277
 278         repo_dir = os.path.join(self.gitroot, 'server_new.export')
 279         with repo.Repo(repo_dir) as dest:
 280             self.assertDictEqual(dest.refs.as_dict(), refs)
 281
 282
 283 class DulwichTCPClientTest(CompatTestCase, DulwichClientTestBase):
 284
 285     def setUp(self):
 286         CompatTestCase.setUp(self)
 287         DulwichClientTestBase.setUp(self)
 288         if check_for_daemon(limit=1):
 289             raise SkipTest('git-daemon was already running on port %s' %
 290                            protocol.TCP_GIT_PORT)
 291         fd, self.pidfile = tempfile.mkstemp(prefix='dulwich-test-git-client',
 292                                             suffix=".pid")
 293         os.fdopen(fd).close()
 294         args = [_DEFAULT_GIT, 'daemon', '--verbose', '--export-all',
 295                 '--pid-file=%s' % self.pidfile,
 296                 '--base-path=%s' % self.gitroot,
 297                 '--enable=receive-pack', '--enable=upload-archive',
 298                 '--listen=localhost', '--reuseaddr',
 299                 self.gitroot]
 300         self.process = subprocess.Popen(
 301             args, cwd=self.gitroot,
 302             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 303         if not check_for_daemon():
 304             raise SkipTest('git-daemon failed to start')
 305
 306     def tearDown(self):
 307         with open(self.pidfile) as f:
 308             pid = int(f.read().strip())
 309         if sys.platform == 'win32':
 310             PROCESS_TERMINATE = 1
 311             handle = ctypes.windll.kernel32.OpenProcess(
 312                 PROCESS_TERMINATE, False, pid)
 313             ctypes.windll.kernel32.TerminateProcess(handle, -1)
 314             ctypes.windll.kernel32.CloseHandle(handle)
 315         else:
 316             try:
 317                 os.kill(pid, signal.SIGKILL)
 318                 os.unlink(self.pidfile)
 319             except (OSError, IOError):
 320                 pass
 321         self.process.wait()
 322         self.process.stdout.close()
 323         self.process.stderr.close()
 324         DulwichClientTestBase.tearDown(self)
 325         CompatTestCase.tearDown(self)
 326
 327     def _client(self):
 328         return client.TCPGitClient('localhost')
 329
 330     def _build_path(self, path):
 331         return path
 332
 333     if sys.platform == 'win32':
 334         @expectedFailure
 335         def test_fetch_pack_no_side_band_64k(self):
 336             DulwichClientTestBase.test_fetch_pack_no_side_band_64k(self)
 337
 338
 339 class TestSSHVendor(object):
 340
 341     @staticmethod
 342     def run_command(host, command, username=None, port=None,
 343                     password=None, key_filename=None):
 344         cmd, path = command.split(' ')
 345         cmd = cmd.split('-', 1)
 346         path = path.replace("'", "")
 347         p = subprocess.Popen(cmd + [path], bufsize=0, stdin=subprocess.PIPE,
 348                              stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 349         return client.SubprocessWrapper(p)
 350
 351
 352 class DulwichMockSSHClientTest(CompatTestCase, DulwichClientTestBase):
 353
 354     def setUp(self):
 355         CompatTestCase.setUp(self)
 356         DulwichClientTestBase.setUp(self)
 357         self.real_vendor = client.get_ssh_vendor
 358         client.get_ssh_vendor = TestSSHVendor
 359
 360     def tearDown(self):
 361         DulwichClientTestBase.tearDown(self)
 362         CompatTestCase.tearDown(self)
 363         client.get_ssh_vendor = self.real_vendor
 364
 365     def _client(self):
 366         return client.SSHGitClient('localhost')
 367
 368     def _build_path(self, path):
 369         return self.gitroot + path
 370
 371
 372 class DulwichSubprocessClientTest(CompatTestCase, DulwichClientTestBase):
 373
 374     def setUp(self):
 375         CompatTestCase.setUp(self)
 376         DulwichClientTestBase.setUp(self)
 377
 378     def tearDown(self):
 379         DulwichClientTestBase.tearDown(self)
 380         CompatTestCase.tearDown(self)
 381
 382     def _client(self):
 383         return client.SubprocessGitClient(stderr=subprocess.PIPE)
 384
 385     def _build_path(self, path):
 386         return self.gitroot + path
 387
 388
 389 class GitHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
 390     """HTTP Request handler that calls out to 'git http-backend'."""
 391
 392     # Make rfile unbuffered -- we need to read one line and then pass
 393     # the rest to a subprocess, so we can't use buffered input.
 394     rbufsize = 0
 395
 396     def do_POST(self):
 397         self.run_backend()
 398
 399     def do_GET(self):
 400         self.run_backend()
 401
 402     def send_head(self):
 403         return self.run_backend()
 404
 405     def log_request(self, code='-', size='-'):
 406         # Let's be quiet, the test suite is noisy enough already
 407         pass
 408
 409     def run_backend(self):
 410         """Call out to git http-backend."""
 411         # Based on CGIHTTPServer.CGIHTTPRequestHandler.run_cgi:
 412         # Copyright (c) 2001-2010 Python Software Foundation;
 413         # All Rights Reserved
 414         # Licensed under the Python Software Foundation License.
 415         rest = self.path
 416         # find an explicit query string, if present.
 417         i = rest.rfind('?')
 418         if i >= 0:
 419             rest, query = rest[:i], rest[i+1:]
 420         else:
 421             query = ''
 422
 423         env = copy.deepcopy(os.environ)
 424         env['SERVER_SOFTWARE'] = self.version_string()
 425         env['SERVER_NAME'] = self.server.server_name
 426         env['GATEWAY_INTERFACE'] = 'CGI/1.1'
 427         env['SERVER_PROTOCOL'] = self.protocol_version
 428         env['SERVER_PORT'] = str(self.server.server_port)
 429         env['GIT_PROJECT_ROOT'] = self.server.root_path
 430         env["GIT_HTTP_EXPORT_ALL"] = "1"
 431         env['REQUEST_METHOD'] = self.command
 432         uqrest = unquote(rest)
 433         env['PATH_INFO'] = uqrest
 434         env['SCRIPT_NAME'] = "/"
 435         if query:
 436             env['QUERY_STRING'] = query
 437         host = self.address_string()
 438         if host != self.client_address[0]:
 439             env['REMOTE_HOST'] = host
 440         env['REMOTE_ADDR'] = self.client_address[0]
 441         authorization = self.headers.get("authorization")
 442         if authorization:
 443             authorization = authorization.split()
 444             if len(authorization) == 2:
 445                 import base64
 446                 import binascii
 447                 env['AUTH_TYPE'] = authorization[0]
 448                 if authorization[0].lower() == "basic":
 449                     try:
 450                         authorization = base64.decodestring(authorization[1])
 451                     except binascii.Error:
 452                         pass
 453                     else:
 454                         authorization = authorization.split(':')
 455                         if len(authorization) == 2:
 456                             env['REMOTE_USER'] = authorization[0]
 457         # XXX REMOTE_IDENT
 458         content_type = self.headers.get('content-type')
 459         if content_type:
 460             env['CONTENT_TYPE'] = content_type
 461         length = self.headers.get('content-length')
 462         if length:
 463             env['CONTENT_LENGTH'] = length
 464         referer = self.headers.get('referer')
 465         if referer:
 466             env['HTTP_REFERER'] = referer
 467         accept = []
 468         for line in self.headers.getallmatchingheaders('accept'):
 469             if line[:1] in "\t\n\r ":
 470                 accept.append(line.strip())
 471             else:
 472                 accept = accept + line[7:].split(',')
 473         env['HTTP_ACCEPT'] = ','.join(accept)
 474         ua = self.headers.get('user-agent')
 475         if ua:
 476             env['HTTP_USER_AGENT'] = ua
 477         co = self.headers.get('cookie')
 478         if co:
 479             env['HTTP_COOKIE'] = co
 480         # XXX Other HTTP_* headers
 481         # Since we're setting the env in the parent, provide empty
 482         # values to override previously set values
 483         for k in ('QUERY_STRING', 'REMOTE_HOST', 'CONTENT_LENGTH',
 484                   'HTTP_USER_AGENT', 'HTTP_COOKIE', 'HTTP_REFERER'):
 485             env.setdefault(k, "")
 486
 487         self.wfile.write(b"HTTP/1.1 200 Script output follows\r\n")
 488         self.wfile.write(
 489             ("Server: %s\r\n" % self.server.server_name).encode('ascii'))
 490         self.wfile.write(
 491             ("Date: %s\r\n" % self.date_time_string()).encode('ascii'))
 492
 493         decoded_query = query.replace('+', ' ')
 494
 495         try:
 496             nbytes = int(length)
 497         except (TypeError, ValueError):
 498             nbytes = 0
 499         if self.command.lower() == "post" and nbytes > 0:
 500             data = self.rfile.read(nbytes)
 501         else:
 502             data = None
 503         # throw away additional data [see bug #427345]
 504         while select.select([self.rfile._sock], [], [], 0)[0]:
 505             if not self.rfile._sock.recv(1):
 506                 break
 507         args = ['http-backend']
 508         if '=' not in decoded_query:
 509             args.append(decoded_query)
 510         stdout = run_git_or_fail(
 511             args, input=data, env=env, stderr=subprocess.PIPE)
 512         self.wfile.write(stdout)
 513
 514
 515 class HTTPGitServer(BaseHTTPServer.HTTPServer):
 516
 517     allow_reuse_address = True
 518
 519     def __init__(self, server_address, root_path):
 520         BaseHTTPServer.HTTPServer.__init__(
 521             self, server_address, GitHTTPRequestHandler)
 522         self.root_path = root_path
 523         self.server_name = "localhost"
 524
 525     def get_url(self):
 526         return 'http://%s:%s/' % (self.server_name, self.server_port)
 527
 528
 529 class DulwichHttpClientTest(CompatTestCase, DulwichClientTestBase):
 530
 531     min_git_version = (1, 7, 0, 2)
 532
 533     def setUp(self):
 534         CompatTestCase.setUp(self)
 535         DulwichClientTestBase.setUp(self)
 536         self._httpd = HTTPGitServer(("localhost", 0), self.gitroot)
 537         self.addCleanup(self._httpd.shutdown)
 538         threading.Thread(target=self._httpd.serve_forever).start()
 539         run_git_or_fail(['config', 'http.uploadpack', 'true'],
 540                         cwd=self.dest)
 541         run_git_or_fail(['config', 'http.receivepack', 'true'],
 542                         cwd=self.dest)
 543
 544     def tearDown(self):
 545         DulwichClientTestBase.tearDown(self)
 546         CompatTestCase.tearDown(self)
 547         self._httpd.shutdown()
 548         self._httpd.socket.close()
 549
 550     def _client(self):
 551         return client.HttpGitClient(self._httpd.get_url())
 552
 553     def _build_path(self, path):
 554         return path
 555
 556     def test_archive(self):
 557         raise SkipTest("exporting archives not supported over http")