var app = angular.module('plunker', []);

app.directive('ngXlinkHref', function () {
  return {
    priority: 99,
    restrict: 'A',
    link: function (scope, element, attr) {
      var attrName = 'xlink:href';
      attr.$observe('ngXlinkHref', function (value) {
        if (!value)
          return;

        attr.$set(attrName, value);
      });
    }
  };
});


app.controller('MainCtrl', function($scope) {
  $scope.mymodel = {
    imageSrc: 'http://placehold.it/500x500'
  };

});
<!doctype html>
<html ng-app="plunker" >
<head>
  <meta charset="utf-8">
  <title>AngularJS Plunker</title>
  <script>document.write('<base href="' + document.location + '" />');</script>
  <link rel="stylesheet" href="style.css">
  <script src="https://ajax.googleapis.com/ajax/libs/angularjs/1.0.5/angular.js"></script>
  <script src="app.js"></script>
</head>
<body ng-controller="MainCtrl">
  <svg id="viewport" class="viewport" width="100%" height="100%" viewBox="0 0 500 500" preserveAspectRatio="xMidYMid meet" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
    <image x="0" y="0" width="500" height="500" ng-xlink-href="{{mymodel.imageSrc}}" xlink:href="" />
  
    <!-- If you remove the xlink:href="" the image will not render, how do we fix this? -->
  </svg>
</body>
</html>
/* Put your css in here */